center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat: 支持API數據機

Browse Source
This commit is contained in:
taojinlong 2022-02-18 17:59:19 +08:00
parent 9fc6cedca3
commit 1bc642dd27
52 changed files with 3792 additions and 320 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/pom.xml
View File

@ -268,6 +268,12 @@
<artifactId>jsoup</artifactId>
<version>1.14.3</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>2.4.0</version>
</dependency>
</dependencies>
<build>

31
backend/src/main/java/io/dataease/auth/api/demo.java Normal file
View File

@ -0,0 +1,31 @@
package io.dataease.auth.api;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import com.github.xiaoymin.knife4j.annotations.ApiSupport;
import io.dataease.base.domain.DatasetTableFunction;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.*;
import java.util.List;
@RestController
@RequestMapping("api/data")
public class demo {
@ApiOperation("查询")
@GetMapping("demo")
public Object listByTableId() {
JSONArray jsonArray = new JSONArray();
for(int i=0;i<100;i++){
JSONObject jsonObject = new JSONObject();
for(int j=0;j<10;j++){
jsonObject.set("column" + j, "value"+j);
}
jsonArray.put(jsonObject);
}
return jsonArray;
}
}

3
backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java
View File

@ -14,7 +14,8 @@ public enum DatasourceTypes {
ck("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"),
db2("db2", "db2", "com.ibm.db2.jcc.DB2Driver", "\"", "\"", "\"", "\""),
es("es", "es", "", "\"", "\"", "\"", "\""),
redshift("redshift", "redshift", "org.postgresql.Driver", "\"", "\"", "\"", "\"");
redshift("redshift", "redshift", "org.postgresql.Driver", "\"", "\"", "\"", "\""),
api("api", "api", "", "\"", "\"", "\"", "\"");
private String feature;

22
backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java
View File

@ -1,6 +1,6 @@
package io.dataease.commons.utils;
import io.dataease.dto.datasource.TableFiled;
import io.dataease.dto.datasource.TableField;
import io.dataease.dto.dataset.ExcelSheetData;
import io.dataease.i18n.Translator;
import org.apache.poi.hssf.eventusermodel.*;
@ -94,7 +94,7 @@ public class ExcelXlsReader implements HSSFListener {
@SuppressWarnings("unused")
private String sheetName;
public List<TableFiled> fields = new ArrayList<>();
public List<TableField> fields = new ArrayList<>();
public List<List<String>> data = new ArrayList<>();
public List<ExcelSheetData> totalSheets = new ArrayList<>();
/**
@ -103,11 +103,11 @@ public class ExcelXlsReader implements HSSFListener {
private boolean isDateFormat = false;
public List<TableFiled> getFields() {
public List<TableField> getFields() {
return fields;
}
public void setFields(List<TableFiled> fields) {
public void setFields(List<TableField> fields) {
this.fields = fields;
}
@ -308,13 +308,13 @@ public class ExcelXlsReader implements HSSFListener {
if(curRow == 0){
for (String s : cellList) {
TableFiled tableFiled = new TableFiled();
tableFiled.setFieldType("TEXT");
tableFiled.setFieldSize(65533);
tableFiled.setFieldName(s);
tableFiled.setRemarks(s);
this.fields.add(tableFiled);
totalSheets.get(totalSheets.size() -1).getFields().add(tableFiled);
TableField tableField = new TableField();
tableField.setFieldType("TEXT");
tableField.setFieldSize(65533);
tableField.setFieldName(s);
tableField.setRemarks(s);
this.fields.add(tableField);
totalSheets.get(totalSheets.size() -1).getFields().add(tableField);
}
}

34
backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java
View File

@ -1,5 +1,5 @@
package io.dataease.commons.utils;
import io.dataease.dto.datasource.TableFiled;
import io.dataease.dto.datasource.TableField;
import io.dataease.dto.dataset.ExcelSheetData;
import io.dataease.i18n.Translator;
import org.apache.commons.collections4.CollectionUtils;
@ -111,7 +111,7 @@ public class ExcelXlsxReader extends DefaultHandler {
*/
private StylesTable stylesTable;
public List<TableFiled> fields = new ArrayList<>();
public List<TableField> fields = new ArrayList<>();
public List<List<String>> data = new ArrayList<>();
public List<ExcelSheetData> totalSheets = new ArrayList<>();
/**
@ -120,11 +120,11 @@ public class ExcelXlsxReader extends DefaultHandler {
private boolean isDateFormat = false;
public List<TableFiled> getFields() {
public List<TableField> getFields() {
return fields;
}
public void setFields(List<TableFiled> fields) {
public void setFields(List<TableField> fields) {
this.fields = fields;
}
@ -238,12 +238,12 @@ public class ExcelXlsxReader extends DefaultHandler {
//将单元格内容加入rowlist中，在这之前先去掉字符串前后的空白符
String value = lastIndex.trim();
if(curRow==1){
TableFiled tableFiled = new TableFiled();
tableFiled.setFieldType("TEXT");
tableFiled.setFieldSize(65533);
tableFiled.setFieldName(value);
tableFiled.setRemarks(value);
this.fields.add(tableFiled);
TableField tableField = new TableField();
tableField.setFieldType("TEXT");
tableField.setFieldSize(65533);
tableField.setFieldName(value);
tableField.setRemarks(value);
this.fields.add(tableField);
}
cellList.add(curCol, value);
curCol++;
@ -451,15 +451,15 @@ public class ExcelXlsxReader extends DefaultHandler {
}
private void addField(String columeName, Integer index){
TableFiled tableFiled = new TableFiled();
tableFiled.setFieldType("TEXT");
tableFiled.setFieldSize(65533);
tableFiled.setFieldName(columeName);
tableFiled.setRemarks(columeName);
TableField tableField = new TableField();
tableField.setFieldType("TEXT");
tableField.setFieldSize(65533);
tableField.setFieldName(columeName);
tableField.setRemarks(columeName);
if(index != null){
this.fields.add(index, tableFiled);
this.fields.add(index, tableField);
}else {
this.fields.add(tableFiled);
this.fields.add(tableField);
}
}
private String getType(String thisStr){

20
backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java
View File

@ -2,6 +2,7 @@ package io.dataease.commons.utils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.EntityBuilder;
import org.apache.http.client.entity.UrlEncodedFormEntity;
@ -90,10 +91,10 @@ public class HttpClientUtil {
HttpResponse response = httpClient.execute(httpGet);
HttpEntity entity = response.getEntity();
return EntityUtils.toString(entity, config.getCharset());
return getResponseStr(response, entity, config);
} catch (Exception e) {
logger.error("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
} finally {
try {
httpClient.close();
@ -136,10 +137,10 @@ public class HttpClientUtil {
HttpResponse response = httpClient.execute(httpPost);
HttpEntity entity = response.getEntity();
return EntityUtils.toString(entity, config.getCharset());
return getResponseStr(response, entity, config);
} catch (Exception e) {
logger.error("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
} finally {
try {
httpClient.close();
@ -198,10 +199,10 @@ public class HttpClientUtil {
HttpResponse response = httpClient.execute(httpPost);
HttpEntity entity = response.getEntity();
return EntityUtils.toString(entity, config.getCharset());
return getResponseStr(response, entity, config);
} catch (Exception e) {
logger.error("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
} finally {
try {
httpClient.close();
@ -211,5 +212,10 @@ public class HttpClientUtil {
}
}
private static String getResponseStr(HttpResponse response, HttpEntity entity, HttpClientConfig config) throws Exception{
if(response.getStatusLine().getStatusCode() >= 400){
throw new Exception(EntityUtils.toString(entity, config.getCharset()));
}
return EntityUtils.toString(entity, config.getCharset());
}
}

4
backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java
View File

@ -6,7 +6,7 @@ import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableIncrementalConfig;
import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.controller.response.DataSetDetail;
import io.dataease.dto.datasource.TableFiled;
import io.dataease.dto.datasource.TableField;
import io.dataease.dto.dataset.DataSetTableDTO;
import io.dataease.dto.dataset.ExcelFileData;
import io.dataease.service.dataset.DataSetTableService;
@ -84,7 +84,7 @@ public class DataSetTableController {
@ApiOperation("查询原始字段")
@PostMapping("getFields")
public List<TableFiled> getFields(@RequestBody DatasetTable datasetTable) throws Exception {
public List<TableField> getFields(@RequestBody DatasetTable datasetTable) throws Exception {
return dataSetTableService.getFields(datasetTable);
}

7
backend/src/main/java/io/dataease/controller/datasource/DatasourceController.java
View File

@ -9,6 +9,7 @@ import io.dataease.commons.utils.PageUtils;
import io.dataease.commons.utils.Pager;
import io.dataease.controller.ResultHolder;
import io.dataease.controller.request.DatasourceUnionRequest;
import io.dataease.controller.request.datasource.ApiDefinition;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.dto.datasource.DBTableDTO;
import io.dataease.service.datasource.DatasourceService;
@ -95,5 +96,11 @@ public class DatasourceController {
return datasourceService.getSchema(datasource);
}
@ApiOperation("校验API数据源")
@PostMapping("/checkApiDatasource")
public ApiDefinition checkApiDatasource(@RequestBody ApiDefinition apiDefinition) throws Exception {
return datasourceService.checkApiDatasource(apiDefinition);
}
}

4
backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java
View File

@ -1,7 +1,7 @@
package io.dataease.controller.request.dataset;
import io.dataease.base.domain.DatasetTable;
import io.dataease.dto.datasource.TableFiled;
import io.dataease.dto.datasource.TableField;
import io.dataease.dto.dataset.ExcelSheetData;
import io.swagger.annotations.ApiModelProperty;
import lombok.Getter;
@ -33,7 +33,7 @@ public class DataSetTableRequest extends DatasetTable {
@ApiModelProperty("类型过滤条件集合")
private List<String> typeFilter;
@ApiModelProperty("字段集合")
private List<TableFiled> fields;
private List<TableField> fields;
@ApiModelProperty("excel sheet集合")
private List<ExcelSheetData> sheets;
@ApiModelProperty("是否合并sheet")

20
backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinition.java Normal file
View File

@ -0,0 +1,20 @@
package io.dataease.controller.request.datasource;
import com.alibaba.fastjson.JSONObject;
import io.dataease.base.domain.DatasetTableField;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class ApiDefinition {
private String name;
private String desc;
private String url;
private String method = "GET";
private List<DatasetTableField> fields;
private String request;
private String dataPath;
private List<JSONObject> datas = new ArrayList<>();
}

21
backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinitionRequest.java Normal file
View File

@ -0,0 +1,21 @@
package io.dataease.controller.request.datasource;
import com.alibaba.fastjson.JSONObject;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class ApiDefinitionRequest {
private List<JSONObject> headers = new ArrayList<>();
private JSONObject body = new JSONObject();
private AuthManager authManager = new AuthManager();
@Data
public class AuthManager{
private String password;
private String username;
private String verification = "";
}
}

3
backend/src/main/java/io/dataease/dto/DatasourceDTO.java
View File

@ -1,5 +1,6 @@
package io.dataease.dto;
import com.alibaba.fastjson.JSONArray;
import io.dataease.base.domain.Datasource;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@ -14,5 +15,5 @@ public class DatasourceDTO extends Datasource {
@ApiModelProperty("权限")
private String privileges;
private JSONArray apiConfiguration;
}

4
backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java
View File

@ -1,6 +1,6 @@
package io.dataease.dto.dataset;
import io.dataease.dto.datasource.TableFiled;
import io.dataease.dto.datasource.TableField;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@ -14,7 +14,7 @@ public class ExcelSheetData {
@ApiModelProperty("数据集合")
private List<List<String>> data;
@ApiModelProperty("字段集合")
private List<TableFiled> fields;
private List<TableField> fields;
@ApiModelProperty("是否sheet")
private boolean isSheet = true;
@ApiModelProperty("json数组")

2
backend/src/main/java/io/dataease/dto/datasource/TableFiled.java → backend/src/main/java/io/dataease/dto/datasource/TableField.java
View File

@ -6,7 +6,7 @@ import lombok.Setter;
@Setter
@Getter
public class TableFiled {
public class TableField {
@ApiModelProperty("字段名称")
private String fieldName;
@ApiModelProperty("重新标记")

5
backend/src/main/java/io/dataease/provider/ProviderFactory.java
View File

@ -4,6 +4,7 @@ import io.dataease.commons.constants.DatasourceTypes;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.query.DDLProvider;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.api.ApiProvider;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
@ -25,6 +26,8 @@ public class ProviderFactory implements ApplicationContextAware {
switch (datasourceType) {
case es:
return context.getBean("es", DatasourceProvider.class);
case api:
return context.getBean("api", DatasourceProvider.class);
default:
return context.getBean("jdbc", DatasourceProvider.class);
}
@ -57,6 +60,8 @@ public class ProviderFactory implements ApplicationContextAware {
return context.getBean("hiveQuery", QueryProvider.class);
case db2:
return context.getBean("db2Query", QueryProvider.class);
case api:
return context.getBean("apiQuery", ApiProvider.class);
default:
return context.getBean("mysqlQuery", QueryProvider.class);
}

214
backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java Normal file
View File

@ -0,0 +1,214 @@
package io.dataease.provider.datasource;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.gson.Gson;
import com.jayway.jsonpath.JsonPath;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.commons.utils.HttpClientConfig;
import io.dataease.commons.utils.HttpClientUtil;
import io.dataease.controller.request.datasource.ApiDefinition;
import io.dataease.controller.request.datasource.ApiDefinitionRequest;
import io.dataease.controller.request.datasource.DatasourceRequest;
import io.dataease.dto.datasource.TableDesc;
import io.dataease.dto.datasource.TableField;
import io.dataease.exception.DataEaseException;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHeaders;
import org.springframework.stereotype.Service;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.*;
import java.util.stream.Collectors;
@Service("api")
public class ApiProvider extends DatasourceProvider{
@Override
public List<String[]> getData(DatasourceRequest datasourceRequest) throws Exception {
ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest);
String response = execHttpRequest(apiDefinition);
return fetchResult(response, apiDefinition.getDataPath());
}
@Override
public List<TableDesc> getTables(DatasourceRequest datasourceRequest) throws Exception {
List<TableDesc> tableDescs = new ArrayList<>();
List<ApiDefinition> lists = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class);
for (ApiDefinition apiDefinition : lists) {
TableDesc tableDesc = new TableDesc();
tableDesc.setName(apiDefinition.getName());
tableDesc.setRemark(apiDefinition.getDesc());
tableDescs.add(tableDesc);
}
return tableDescs;
}
@Override
public List<String[]> fetchResult(DatasourceRequest datasourceRequest) throws Exception {
return null;
}
@Override
public List<TableField> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
return null;
}
@Override
public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception {
Map<String, List> result = new HashMap<>();
List<String[]> dataList = new ArrayList<>();
List<TableField> fieldList = new ArrayList<>();;
ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest);
String response = execHttpRequest(apiDefinition);
fieldList = getTableFileds(datasourceRequest);
result.put("fieldList", fieldList);
dataList = fetchResult(response, apiDefinition.getDataPath());
result.put("dataList", dataList);
return result;
}
@Override
public void handleDatasource(DatasourceRequest datasourceRequest, String type) throws Exception {
}
@Override
public List<String> getSchema(DatasourceRequest datasourceRequest) throws Exception {
return null;
}
@Override
public List<TableField> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
List<ApiDefinition> lists = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class);
List<TableField> tableFields = new ArrayList<>();
for (ApiDefinition list : lists) {
if(datasourceRequest.getTable().equalsIgnoreCase(list.getName())){
for (DatasetTableField field : list.getFields()) {
TableField tableField = new TableField();
tableField.setFieldName(field.getOriginName());
tableField.setRemarks(field.getName());
tableField.setFieldSize(field.getSize());
tableField.setFieldType(field.getDeExtractType().toString());
tableFields.add(tableField);
}
}
}
return tableFields;
}
@Override
public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
List<ApiDefinition> apiDefinitionList = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class).stream().filter(item -> item.getName().equalsIgnoreCase(datasourceRequest.getTable())).collect(Collectors.toList());
int success = 0;
for (ApiDefinition apiDefinition : apiDefinitionList) {
datasourceRequest.setTable(apiDefinition.getName());
try {
getData(datasourceRequest);
success++;
}catch (Exception ignore){}
}
if(success == apiDefinitionList.size()){
return "Success";
}
if(success > 0 && success < apiDefinitionList.size() ){
return "Warning";
}
return "Error";
}
static public String execHttpRequest(ApiDefinition apiDefinition) throws Exception{
String response = "";
HttpClientConfig httpClientConfig = new HttpClientConfig();
ApiDefinitionRequest apiDefinitionRequest = JSONObject.parseObject(apiDefinition.getRequest(), ApiDefinitionRequest.class);
System.out.println(new Gson().toJson(apiDefinitionRequest.getAuthManager()));
//headers
for (JSONObject header : apiDefinitionRequest.getHeaders()) {
if(StringUtils.isNotEmpty(header.getString("name")) && StringUtils.isNotEmpty(header.getString("value"))){
httpClientConfig.addHeader(header.getString("name"), header.getString("value"));
}
}
if(apiDefinitionRequest.getAuthManager() != null
&& StringUtils.isNotBlank(apiDefinitionRequest.getAuthManager().getUsername())
&& StringUtils.isNotBlank(apiDefinitionRequest.getAuthManager().getPassword())
&& apiDefinitionRequest.getAuthManager().getVerification().equals("Basic Auth")){
String authValue = "Basic " + Base64.getUrlEncoder().encodeToString((apiDefinitionRequest.getAuthManager().getUsername()
+ ":" + apiDefinitionRequest.getAuthManager().getPassword()).getBytes());
httpClientConfig.addHeader("Authorization", authValue);
}
switch (apiDefinition.getMethod()){
case "GET":
response = HttpClientUtil.get(apiDefinition.getUrl(), httpClientConfig);
break;
case "POST":
if (!apiDefinitionRequest.getBody().containsKey("type")) {
throw new Exception("请求类型不能为空");
}
String type = apiDefinitionRequest.getBody().getString("type");
if (StringUtils.equalsAny(type, "JSON", "XML", "Raw")) {
String raw = null;
if (apiDefinitionRequest.getBody().containsKey("raw")) {
raw = apiDefinitionRequest.getBody().getString("raw");
response = HttpClientUtil.post(apiDefinition.getUrl(), raw, httpClientConfig);
}
}
if (StringUtils.equalsAny(type, "Form_Data", "WWW_FORM")) {
if (apiDefinitionRequest.getBody().containsKey("kvs")) {
Map<String, String> body = new HashMap<>();
JSONArray kvsArr = apiDefinitionRequest.getBody().getJSONArray("kvs");
for (int i = 0; i < kvsArr.size(); i++) {
JSONObject kv = kvsArr.getJSONObject(i);
if (kv.containsKey("name")) {
body.put(kv.getString("name"), kv.getString("value"));
}
}
response = HttpClientUtil.post(apiDefinition.getUrl(), body, httpClientConfig);
}
}
break;
default:
break;
}
System.out.println("response: " + response);
return response;
}
private List<String[]> fetchResult(String result, String path){
List<String[]> dataList = new LinkedList<>();
List<LinkedHashMap> datas = JsonPath.read(result, path);
for (LinkedHashMap data : datas) {
String[] row = new String[data.entrySet().size()];
Iterator it = data.entrySet().iterator();
int i = 0;
while (it.hasNext()){
Map.Entry entry = (Map.Entry)it.next();
row[i] = entry.getValue().toString();
i++;
}
dataList.add(row);
}
return dataList;
}
private ApiDefinition checkApiDefinition(DatasourceRequest datasourceRequest)throws Exception{
List<ApiDefinition> apiDefinitionList = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class).stream().filter(item -> item.getName().equalsIgnoreCase(datasourceRequest.getTable())).collect(Collectors.toList());
if(CollectionUtils.isEmpty(apiDefinitionList)){
throw new Exception("未找到API数据表");
}
if(apiDefinitionList.size() > 1 ){
throw new Exception("存在重名的API数据表");
}
for (ApiDefinition apiDefinition : apiDefinitionList) {
if (apiDefinition.getName().equalsIgnoreCase(datasourceRequest.getTable())){
return apiDefinition;
}
}
throw new Exception("未找到API数据表");
}
}

9
backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java
View File

@ -1,7 +1,7 @@
package io.dataease.provider.datasource;
import io.dataease.dto.datasource.TableDesc;
import io.dataease.dto.datasource.TableFiled;
import io.dataease.dto.datasource.TableField;
import io.dataease.controller.request.datasource.DatasourceRequest;
import java.util.List;
@ -15,13 +15,14 @@ public abstract class DatasourceProvider {
abstract public List<TableDesc> getTables(DatasourceRequest datasourceRequest) throws Exception;
public void checkStatus(DatasourceRequest datasourceRequest) throws Exception {
public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
getData(datasourceRequest);
return "Success";
}
abstract public List<String[]> fetchResult(DatasourceRequest datasourceRequest) throws Exception;
abstract public List<TableFiled> fetchResultField(DatasourceRequest datasourceRequest) throws Exception;
abstract public List<TableField> fetchResultField(DatasourceRequest datasourceRequest) throws Exception;
abstract public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception;
@ -29,5 +30,5 @@ public abstract class DatasourceProvider {
abstract public List<String> getSchema(DatasourceRequest datasourceRequest) throws Exception;
public abstract List<TableFiled> getTableFileds(DatasourceRequest datasourceRequest) throws Exception;
public abstract List<TableField> getTableFileds(DatasourceRequest datasourceRequest) throws Exception;
}

27
backend/src/main/java/io/dataease/provider/datasource/EsProvider.java
View File

@ -10,7 +10,7 @@ import io.dataease.controller.request.datasource.es.RequstWithCursor;
import io.dataease.controller.request.datasource.DatasourceRequest;
import io.dataease.dto.datasource.EsConfiguration;
import io.dataease.dto.datasource.TableDesc;
import io.dataease.dto.datasource.TableFiled;
import io.dataease.dto.datasource.TableField;
import io.dataease.exception.DataEaseException;
import io.dataease.i18n.Translator;
import io.dataease.provider.ProviderFactory;
@ -98,7 +98,7 @@ public class EsProvider extends DatasourceProvider {
}
@Override
public List<TableFiled> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
public List<TableField> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
QueryProvider qp = ProviderFactory.getQueryProvider(datasourceRequest.getDatasource().getType());
datasourceRequest.setQuery(qp.convertTableToSql(datasourceRequest.getTable(), datasourceRequest.getDatasource()));
return fetchResultField(datasourceRequest);
@ -119,26 +119,26 @@ public class EsProvider extends DatasourceProvider {
}
@Override
public List<TableFiled> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
List<TableFiled> tableFileds = new ArrayList<>();
public List<TableField> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
List<TableField> tableFields = new ArrayList<>();
try {
String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json");
tableFileds = fetchResultField4Sql(response);
tableFields = fetchResultField4Sql(response);
} catch (Exception e) {
DataEaseException.throwException(e);
}
return tableFileds;
return tableFields;
}
private List<TableFiled> fetchResultField(String response) throws Exception {
List<TableFiled> fieldList = new ArrayList<>();
private List<TableField> fetchResultField(String response) throws Exception {
List<TableField> fieldList = new ArrayList<>();
EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
if (esReponse.getError() != null) {
throw new Exception(esReponse.getError().getReason());
}
for (String[] row : esReponse.getRows()) {
TableFiled field = new TableFiled();
TableField field = new TableField();
field.setFieldName(row[0]);
field.setRemarks(row[0]);
field.setFieldType(row[2]);
@ -148,15 +148,15 @@ public class EsProvider extends DatasourceProvider {
return fieldList;
}
private List<TableFiled> fetchResultField4Sql(String response) throws Exception {
List<TableFiled> fieldList = new ArrayList<>();
private List<TableField> fetchResultField4Sql(String response) throws Exception {
List<TableField> fieldList = new ArrayList<>();
EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
if (esReponse.getError() != null) {
throw new Exception(esReponse.getError().getReason());
}
for (EsReponse.Column column : esReponse.getColumns()) {
TableFiled field = new TableFiled();
TableField field = new TableField();
field.setFieldName(column.getName());
field.setRemarks(column.getName());
field.setFieldType(column.getType());
@ -225,7 +225,7 @@ public class EsProvider extends DatasourceProvider {
@Override
public void checkStatus(DatasourceRequest datasourceRequest) throws Exception {
public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class);
String response = exexGetQuery(datasourceRequest);
@ -246,6 +246,7 @@ public class EsProvider extends DatasourceProvider {
}
datasourceRequest.getDatasource().setConfiguration(new Gson().toJson(esConfiguration));
getTables(datasourceRequest);
return "Success";
}
private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri) {

55
backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
View File

@ -138,12 +138,12 @@ public class JdbcProvider extends DatasourceProvider {
}
@Override
public List<TableFiled> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
public List<TableField> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
if(datasourceRequest.getDatasource().getType().equalsIgnoreCase("mongo")){
datasourceRequest.setQuery("select * from " + datasourceRequest.getTable());
return fetchResultField(datasourceRequest);
}
List<TableFiled> list = new LinkedList<>();
List<TableField> list = new LinkedList<>();
try (Connection connection = getConnectionFromPool(datasourceRequest)) {
if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("oracle")) {
Method setRemarksReporting = extendedJdbcClassLoader.loadClass("oracle.jdbc.driver.OracleConnection").getMethod("setRemarksReporting",boolean.class);
@ -161,13 +161,13 @@ public class JdbcProvider extends DatasourceProvider {
}
if (database != null) {
if (tableName.equals(datasourceRequest.getTable()) && database.equalsIgnoreCase(getDatabase(datasourceRequest))) {
TableFiled tableFiled = getTableFiled(resultSet, datasourceRequest);
list.add(tableFiled);
TableField tableField = getTableFiled(resultSet, datasourceRequest);
list.add(tableField);
}
} else {
if (tableName.equals(datasourceRequest.getTable())) {
TableFiled tableFiled = getTableFiled(resultSet, datasourceRequest);
list.add(tableFiled);
TableField tableField = getTableFiled(resultSet, datasourceRequest);
list.add(tableField);
}
}
}
@ -186,40 +186,40 @@ public class JdbcProvider extends DatasourceProvider {
return list;
}
private TableFiled getTableFiled(ResultSet resultSet, DatasourceRequest datasourceRequest) throws SQLException {
TableFiled tableFiled = new TableFiled();
private TableField getTableFiled(ResultSet resultSet, DatasourceRequest datasourceRequest) throws SQLException {
TableField tableField = new TableField();
String colName = resultSet.getString("COLUMN_NAME");
tableFiled.setFieldName(colName);
tableField.setFieldName(colName);
String remarks = resultSet.getString("REMARKS");
if (remarks == null || remarks.equals("")) {
remarks = colName;
}
tableFiled.setRemarks(remarks);
tableField.setRemarks(remarks);
String dbType = resultSet.getString("TYPE_NAME").toUpperCase();
tableFiled.setFieldType(dbType);
tableField.setFieldType(dbType);
if (dbType.equalsIgnoreCase("LONG")) {
tableFiled.setFieldSize(65533);
tableField.setFieldSize(65533);
}
if (StringUtils.isNotEmpty(dbType) && dbType.toLowerCase().contains("date") && tableFiled.getFieldSize() < 50) {
tableFiled.setFieldSize(50);
if (StringUtils.isNotEmpty(dbType) && dbType.toLowerCase().contains("date") && tableField.getFieldSize() < 50) {
tableField.setFieldSize(50);
}
if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name())) {
QueryProvider qp = ProviderFactory.getQueryProvider(datasourceRequest.getDatasource().getType());
tableFiled.setFieldSize(qp.transFieldSize(dbType));
tableField.setFieldSize(qp.transFieldSize(dbType));
} else {
if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.hive.name()) && tableFiled.getFieldType().equalsIgnoreCase("BOOLEAN")) {
tableFiled.setFieldSize(1);
if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.hive.name()) && tableField.getFieldType().equalsIgnoreCase("BOOLEAN")) {
tableField.setFieldSize(1);
} else {
String size = resultSet.getString("COLUMN_SIZE");
if (size == null) {
tableFiled.setFieldSize(1);
tableField.setFieldSize(1);
} else {
tableFiled.setFieldSize(Integer.valueOf(size));
tableField.setFieldSize(Integer.valueOf(size));
}
}
}
return tableFiled;
return tableField;
}
private String getDatabase(DatasourceRequest datasourceRequest) {
@ -244,7 +244,7 @@ public class JdbcProvider extends DatasourceProvider {
}
@Override
public List<TableFiled> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
public List<TableField> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(rebuildSqlWithFragment(datasourceRequest.getQuery()))) {
return fetchResultField(rs, datasourceRequest);
} catch (SQLException e) {
@ -259,7 +259,7 @@ public class JdbcProvider extends DatasourceProvider {
public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception {
Map<String, List> result = new HashMap<>();
List<String[]> dataList;
List<TableFiled> fieldList;
List<TableField> fieldList;
try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(rebuildSqlWithFragment(datasourceRequest.getQuery()))) {
fieldList = fetchResultField(rs, datasourceRequest);
result.put("fieldList", fieldList);
@ -274,8 +274,8 @@ public class JdbcProvider extends DatasourceProvider {
return new HashMap<>();
}
private List<TableFiled> fetchResultField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
List<TableFiled> fieldList = new ArrayList<>();
private List<TableField> fetchResultField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
List<TableField> fieldList = new ArrayList<>();
ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount();
for (int j = 0; j < columnCount; j++) {
@ -285,7 +285,7 @@ public class JdbcProvider extends DatasourceProvider {
if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.hive.name()) && l.contains("\\.")) {
l = l.split("\\.")[1];
}
TableFiled field = new TableFiled();
TableField field = new TableField();
field.setFieldName(l);
field.setRemarks(l);
field.setFieldType(t);
@ -362,14 +362,15 @@ public class JdbcProvider extends DatasourceProvider {
}
@Override
public void checkStatus(DatasourceRequest datasourceRequest) throws Exception {
public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
String queryStr = getTablesSql(datasourceRequest);
try (Connection con = getConnection(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryStr)) {
return "Success";
} catch (Exception e) {
e.printStackTrace();
DataEaseException.throwException(e.getMessage());
}
return "Error";
}
@Override

122
backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java Normal file
View File

@ -0,0 +1,122 @@
package io.dataease.provider.query;
import io.dataease.base.domain.ChartViewWithBLOBs;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.Datasource;
import io.dataease.controller.request.chart.ChartExtFilterRequest;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import java.util.List;
public class QueryProviderImpl extends QueryProvider {
@Override
public Integer transFieldType(String field) {
return null;
}
@Override
public String createSQLPreview(String sql, String orderBy) {
return null;
}
@Override
public String createQuerySQL(String table, List<DatasetTableField> fields, boolean isGroup, Datasource ds, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
return null;
}
@Override
public String createQuerySQLAsTmp(String sql, List<DatasetTableField> fields, boolean isGroup, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
return null;
}
@Override
public String createQueryTableWithPage(String table, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
return null;
}
@Override
public String createQuerySQLWithPage(String sql, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
return null;
}
@Override
public String createQueryTableWithLimit(String table, List<DatasetTableField> fields, Integer limit, boolean isGroup, Datasource ds, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
return null;
}
@Override
public String createQuerySqlWithLimit(String sql, List<DatasetTableField> fields, Integer limit, boolean isGroup, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
return null;
}
@Override
public String getSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
return null;
}
@Override
public String getSQLAsTmp(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, ChartViewWithBLOBs view) {
return null;
}
@Override
public String getSQLTableInfo(String table, List<ChartViewFieldDTO> xAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
return null;
}
@Override
public String getSQLAsTmpTableInfo(String sql, List<ChartViewFieldDTO> xAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
return null;
}
@Override
public String getSQLStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack, Datasource ds, ChartViewWithBLOBs view) {
return null;
}
@Override
public String getSQLAsTmpStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack, ChartViewWithBLOBs view) {
return null;
}
@Override
public String getSQLScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble, Datasource ds, ChartViewWithBLOBs view) {
return null;
}
@Override
public String getSQLAsTmpScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble, ChartViewWithBLOBs view) {
return null;
}
@Override
public String searchTable(String table) {
return null;
}
@Override
public String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, ChartViewWithBLOBs view) {
return null;
}
@Override
public String getSQLSummaryAsTmp(String sql, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, ChartViewWithBLOBs view) {
return null;
}
@Override
public String wrapSql(String sql) {
return null;
}
@Override
public String createRawQuerySQL(String table, List<DatasetTableField> fields, Datasource ds) {
return null;
}
@Override
public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
return null;
}
}

25
backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java Normal file
View File

@ -0,0 +1,25 @@
package io.dataease.provider.query.api;
import io.dataease.provider.query.QueryProviderImpl;
import org.springframework.stereotype.Service;
@Service("apiQuery")
public class ApiProvider extends QueryProviderImpl {
@Override
public Integer transFieldType(String field) {
switch (field) {
case "0":
return 0;// 文本
case "1":
return 1;// 时间
case "2":
return 2;// 整型
case "3":
return 3;// 浮点
case "4":
return 4;// 布尔
default:
return 0;
}
}
}

10
backend/src/main/java/io/dataease/service/chart/ChartViewService.java
View File

@ -242,9 +242,9 @@ public class ChartViewService {
//列权限
List<String> desensitizationList = new ArrayList<>();
fields = permissionService.filterColumnPermissons(fields, desensitizationList, datasetTable.getId(), requestList.getUser());
List<DatasetTableField> columnPermissionFields = permissionService.filterColumnPermissons(fields, desensitizationList, datasetTable.getId(), requestList.getUser());
//将没有权限的列删掉
List<String> dataeaseNames = fields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList());
List<String> dataeaseNames = columnPermissionFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList());
dataeaseNames.add("*");
fieldCustomFilter = fieldCustomFilter.stream().filter(item -> !desensitizationList.contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList());
extStack = extStack.stream().filter(item -> !desensitizationList.contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList());
@ -253,8 +253,8 @@ public class ChartViewService {
//行权限
List<ChartFieldCustomFilterDTO> permissionFields = permissionService.getCustomFilters(fields, datasetTable, requestList.getUser());
fieldCustomFilter.addAll(permissionFields);
List<ChartFieldCustomFilterDTO> rowPermissionFields = permissionService.getCustomFilters(columnPermissionFields, datasetTable, requestList.getUser());
fieldCustomFilter.addAll(rowPermissionFields);
for (ChartFieldCustomFilterDTO ele : fieldCustomFilter) {
ele.setField(dataSetTableFieldsService.get(ele.getId()));
@ -485,7 +485,7 @@ public class ChartViewService {
// 仪表板有参数不实用缓存
if (!cache || CollectionUtils.isNotEmpty(requestList.getFilter())
|| CollectionUtils.isNotEmpty(requestList.getLinkageFilters())
|| CollectionUtils.isNotEmpty(requestList.getDrill())) {
|| CollectionUtils.isNotEmpty(requestList.getDrill()) || CollectionUtils.isNotEmpty(rowPermissionFields) || fields.size() != columnPermissionFields.size()) {
data = datasourceProvider.getData(datasourceRequest);
} else {
try {

157
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -19,7 +19,7 @@ import io.dataease.dto.dataset.*;
import io.dataease.dto.dataset.union.UnionDTO;
import io.dataease.dto.dataset.union.UnionItemDTO;
import io.dataease.dto.dataset.union.UnionParamDTO;
import io.dataease.dto.datasource.TableFiled;
import io.dataease.dto.datasource.TableField;
import io.dataease.exception.DataEaseException;
import io.dataease.i18n.Translator;
import io.dataease.plugins.loader.ClassloaderResponsity;
@ -143,7 +143,7 @@ public class DataSetTableService {
sheetTable.setName(excelSheetDataList.get(0).getDatasetName());
checkName(sheetTable);
excelSheetDataList.forEach(excelSheetData -> {
String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName)
String[] fieldArray = excelSheetData.getFields().stream().map(TableField::getFieldName)
.toArray(String[]::new);
if (checkIsRepeat(fieldArray)) {
DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
@ -164,7 +164,7 @@ public class DataSetTableService {
});
} else {
for (ExcelSheetData sheet : datasetTable.getSheets()) {
String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName)
String[] fieldArray = sheet.getFields().stream().map(TableField::getFieldName)
.toArray(String[]::new);
if (checkIsRepeat(fieldArray)) {
DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
@ -197,12 +197,12 @@ public class DataSetTableService {
}
List<ExcelSheetData> excelSheetDataList = new ArrayList<>();
List<String> oldFields = datasetTable.getSheets().get(0).getFields().stream().map(TableFiled::getRemarks)
List<String> oldFields = datasetTable.getSheets().get(0).getFields().stream().map(TableField::getRemarks)
.collect(Collectors.toList());
for (ExcelSheetData sheet : datasetTable.getSheets()) {
// 替换时，
if (datasetTable.getEditType() == 0) {
List<String> newFields = sheet.getFields().stream().map(TableFiled::getRemarks)
List<String> newFields = sheet.getFields().stream().map(TableField::getRemarks)
.collect(Collectors.toList());
if (!oldFields.equals(newFields)) {
DataEaseException.throwException(Translator.get("i18n_excel_column_inconsistent"));
@ -210,7 +210,7 @@ public class DataSetTableService {
oldFields = newFields;
}
String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new);
String[] fieldArray = sheet.getFields().stream().map(TableField::getFieldName).toArray(String[]::new);
if (checkIsRepeat(fieldArray)) {
DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
}
@ -405,7 +405,7 @@ public class DataSetTableService {
return extDataSetTableMapper.searchOne(dataSetTableRequest);
}
public List<TableFiled> getFields(DatasetTable datasetTable) throws Exception {
public List<TableField> getFields(DatasetTable datasetTable) throws Exception {
Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
@ -496,7 +496,7 @@ public class DataSetTableService {
if (page == Integer.parseInt(dataSetTableRequest.getRow()) / pageSize + 1) {
realSize = Integer.parseInt(dataSetTableRequest.getRow()) % pageSize;
}
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db")) {
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "api")) {
if (datasetTable.getMode() == 0) {
Datasource ds = datasourceMapper.selectByPrimaryKey(dataSetTableRequest.getDataSourceId());
if (ObjectUtils.isEmpty(ds)) {
@ -847,8 +847,8 @@ public class DataSetTableService {
datasourceRequest.setQuery(sqlAsTable);
Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
List<String[]> data = result.get("dataList");
List<TableFiled> fields = result.get("fieldList");
String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new);
List<TableField> fields = result.get("fieldList");
String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new);
if (checkIsRepeat(fieldArray)) {
DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
}
@ -896,8 +896,8 @@ public class DataSetTableService {
datasourceRequest.setQuery(qp.createSQLPreview(sql, null));
Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
List<String[]> data = result.get("dataList");
List<TableFiled> fields = result.get("fieldList");
String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new);
List<TableField> fields = result.get("fieldList");
String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new);
List<Map<String, Object>> jsonArray = new ArrayList<>();
if (CollectionUtils.isNotEmpty(data)) {
@ -912,14 +912,14 @@ public class DataSetTableService {
// 获取每个字段在当前de数据库中的name，作为sql查询后的remarks返回前端展示
for (DatasetTableField datasetTableField : fieldList) {
for (TableFiled tableFiled : fields) {
if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(),
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(tableField.getFieldName(),
DorisTableUtils.dorisFieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
|| StringUtils.equalsIgnoreCase(tableFiled.getFieldName(),
|| StringUtils.equalsIgnoreCase(tableField.getFieldName(),
DorisTableUtils.dorisFieldNameShort(datasetTableField.getTableId() + "_"
+ datasetTableField.getOriginName()))) {
tableFiled.setRemarks(datasetTableField.getName());
tableField.setRemarks(datasetTableField.getName());
break;
}
}
@ -957,8 +957,8 @@ public class DataSetTableService {
datasourceRequest.setQuery(qp.createSQLPreview(sql, null));
Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
List<String[]> data = result.get("dataList");
List<TableFiled> fields = result.get("fieldList");
String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new);
List<TableField> fields = result.get("fieldList");
String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new);
List<Map<String, Object>> jsonArray = new ArrayList<>();
if (CollectionUtils.isNotEmpty(data)) {
@ -976,13 +976,13 @@ public class DataSetTableService {
dataTableInfoDTO.getList().forEach(
ele -> checkedFieldList.addAll(dataSetTableFieldsService.getListByIds(ele.getCheckedFields())));
for (DatasetTableField datasetTableField : checkedFieldList) {
for (TableFiled tableFiled : fields) {
if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(),
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(tableField.getFieldName(),
DorisTableUtils.dorisFieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
|| StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldName(
|| StringUtils.equalsIgnoreCase(tableField.getFieldName(), DorisTableUtils.dorisFieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) {
tableFiled.setRemarks(datasetTableField.getName());
tableField.setRemarks(datasetTableField.getName());
break;
}
}
@ -1417,11 +1417,11 @@ public class DataSetTableService {
}
}
public List<DatasetTableField> saveExcelTableField(String datasetTableId, List<TableFiled> fields, boolean insert) {
public List<DatasetTableField> saveExcelTableField(String datasetTableId, List<TableField> fields, boolean insert) {
List<DatasetTableField> datasetTableFields = new ArrayList<>();
if (CollectionUtils.isNotEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
TableFiled filed = fields.get(i);
TableField filed = fields.get(i);
DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(datasetTableId);
datasetTableField.setOriginName(filed.getFieldName());
@ -1450,9 +1450,9 @@ public class DataSetTableService {
DataSetTableRequest dataSetTableRequest = new DataSetTableRequest();
BeanUtils.copyBean(dataSetTableRequest, datasetTable);
List<TableFiled> fields = new ArrayList<>();
List<TableField> fields = new ArrayList<>();
long syncTime = System.currentTimeMillis();
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db")) {
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "api")) {
fields = getFields(datasetTable);
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql")) {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
@ -1514,11 +1514,11 @@ public class DataSetTableService {
fieldList.addAll(listByIds);
});
for (DatasetTableField field : fieldList) {
for (TableFiled tableFiled : fields) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getOriginName()),
tableFiled.getFieldName())) {
tableFiled.setRemarks(field.getName());
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
}
}
@ -1544,11 +1544,11 @@ public class DataSetTableService {
datasourceRequest.setQuery(sql);
fields = datasourceProvider.fetchResultField(datasourceRequest);
for (DatasetTableField field : fieldList) {
for (TableFiled tableFiled : fields) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getDataeaseName()),
tableFiled.getFieldName())) {
tableFiled.setRemarks(field.getName());
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
}
}
@ -1568,11 +1568,11 @@ public class DataSetTableService {
fields = datasourceProvider.fetchResultField(datasourceRequest);
for (DatasetTableField field : fieldList) {
for (TableFiled tableFiled : fields) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldNameShort(field.getTableId() + "_" + field.getOriginName()),
tableFiled.getFieldName())) {
tableFiled.setRemarks(field.getName());
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
}
}
@ -1586,16 +1586,14 @@ public class DataSetTableService {
if (CollectionUtils.isNotEmpty(fields)) {
List<String> originNameList = new ArrayList<>();
for (int i = 0; i < fields.size(); i++) {
TableFiled filed = fields.get(i);
TableField filed = fields.get(i);
originNameList.add(filed.getFieldName());
DatasetTableField datasetTableField = DatasetTableField.builder().build();
// 物理字段名设定为唯一，查询当前数据集下是否已存在该字段，存在则update，不存在则insert
DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample();
// 字段名一致，认为字段没有改变
datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId())
.andOriginNameEqualTo(filed.getFieldName());
List<DatasetTableField> datasetTableFields = datasetTableFieldMapper
.selectByExample(datasetTableFieldExample);
datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andOriginNameEqualTo(filed.getFieldName());
List<DatasetTableField> datasetTableFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample);
if (CollectionUtils.isNotEmpty(datasetTableFields)) {
datasetTableField.setId(datasetTableFields.get(0).getId());
datasetTableField.setOriginName(filed.getFieldName());
@ -1636,8 +1634,7 @@ public class DataSetTableService {
}
// delete 数据库中多余的字段
DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample();
datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andExtFieldEqualTo(0)
.andOriginNameNotIn(originNameList);
datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andExtFieldEqualTo(0).andOriginNameNotIn(originNameList);
datasetTableFieldMapper.deleteByExample(datasetTableFieldExample);
}
}
@ -1736,7 +1733,7 @@ public class DataSetTableService {
datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> sqlFileds = new ArrayList<>();
try {
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName)
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName)
.forEach(filed -> {
sqlFileds.add(filed);
});
@ -1756,7 +1753,7 @@ public class DataSetTableService {
datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> sqlFileds = new ArrayList<>();
try {
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName)
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName)
.forEach(filed -> sqlFileds.add(filed));
} catch (Exception e) {
DataEaseException.throwException(Translator.get("i18n_check_sql_error") + e.getMessage());
@ -1820,8 +1817,8 @@ public class DataSetTableService {
List<String> oldFields = datasetTableFields.stream().map(DatasetTableField::getOriginName)
.collect(Collectors.toList());
for (ExcelSheetData excelSheetData : excelSheetDataList) {
List<TableFiled> fields = excelSheetData.getFields();
List<String> newFields = fields.stream().map(TableFiled::getRemarks).collect(Collectors.toList());
List<TableField> fields = excelSheetData.getFields();
List<String> newFields = fields.stream().map(TableField::getRemarks).collect(Collectors.toList());
if (oldFields.equals(newFields)) {
retrunSheetDataList.add(excelSheetData);
}
@ -1879,7 +1876,7 @@ public class DataSetTableService {
inputStream.close();
excelSheetDataList.forEach(excelSheetData -> {
List<List<String>> data = excelSheetData.getData();
String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName)
String[] fieldArray = excelSheetData.getFields().stream().map(TableField::getFieldName)
.toArray(String[]::new);
List<Map<String, Object>> jsonArray = new ArrayList<>();
if (CollectionUtils.isNotEmpty(data)) {
@ -1901,7 +1898,7 @@ public class DataSetTableService {
private Map<String, Object> parseExcel(String filename, InputStream inputStream, boolean isPreview)
throws Exception {
String suffix = filename.substring(filename.lastIndexOf(".") + 1);
List<TableFiled> fields = new ArrayList<>();
List<TableField> fields = new ArrayList<>();
List<String[]> data = new ArrayList<>();
List<Map<String, Object>> jsonArray = new ArrayList<>();
List<String> sheets = new ArrayList<>();
@ -1933,16 +1930,16 @@ public class DataSetTableService {
String[] r = new String[columnNum];
for (int j = 0; j < columnNum; j++) {
if (i == 0) {
TableFiled tableFiled = new TableFiled();
tableFiled.setFieldType("TEXT");
tableFiled.setFieldSize(1024);
TableField tableField = new TableField();
tableField.setFieldType("TEXT");
tableField.setFieldSize(1024);
String columnName = readCell(row.getCell(j), false, null);
if (StringUtils.isEmpty(columnName)) {
columnName = "NONE_" + String.valueOf(j);
}
tableFiled.setFieldName(columnName);
tableFiled.setRemarks(columnName);
fields.add(tableFiled);
tableField.setFieldName(columnName);
tableField.setRemarks(columnName);
fields.add(tableField);
} else {
if (row == null) {
break;
@ -1981,17 +1978,17 @@ public class DataSetTableService {
String[] r = new String[columnNum];
for (int j = 0; j < columnNum; j++) {
if (i == 0) {
TableFiled tableFiled = new TableFiled();
tableFiled.setFieldType("TEXT");
tableFiled.setFieldSize(1024);
TableField tableField = new TableField();
tableField.setFieldType("TEXT");
tableField.setFieldSize(1024);
String columnName = readCell(row.getCell(j), false, null);
if (StringUtils.isEmpty(columnName)) {
columnName = "NONE_" + String.valueOf(j);
}
tableFiled.setFieldName(columnName);
tableFiled.setRemarks(columnName);
fields.add(tableFiled);
tableField.setFieldName(columnName);
tableField.setRemarks(columnName);
fields.add(tableField);
} else {
if (row == null) {
break;
@ -2008,11 +2005,11 @@ public class DataSetTableService {
String s = reader.readLine();// first line
String[] split = s.split(",");
for (String s1 : split) {
TableFiled tableFiled = new TableFiled();
tableFiled.setFieldName(s1);
tableFiled.setRemarks(s1);
tableFiled.setFieldType("TEXT");
fields.add(tableFiled);
TableField tableField = new TableField();
tableField.setFieldName(s1);
tableField.setRemarks(s1);
tableField.setFieldType("TEXT");
fields.add(tableField);
}
int num = 1;
String line;
@ -2027,7 +2024,7 @@ public class DataSetTableService {
}
}
String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new);
String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new);
// 校验excel字段是否重名
if (checkIsRepeat(fieldArray)) {
@ -2052,7 +2049,7 @@ public class DataSetTableService {
return map;
}
private String readCell(Cell cell, boolean cellType, TableFiled tableFiled) {
private String readCell(Cell cell, boolean cellType, TableField tableField) {
if (cell == null) {
return "";
}
@ -2065,15 +2062,15 @@ public class DataSetTableService {
double eps = 1e-10;
if (value - Math.floor(value) < eps) {
if (cellType) {
if (StringUtils.isEmpty(tableFiled.getFieldType())
|| tableFiled.getFieldType().equalsIgnoreCase("TEXT")) {
tableFiled.setFieldType("LONG");
if (StringUtils.isEmpty(tableField.getFieldType())
|| tableField.getFieldType().equalsIgnoreCase("TEXT")) {
tableField.setFieldType("LONG");
}
}
return value.longValue() + "";
} else {
if (cellType) {
tableFiled.setFieldType("DOUBLE");
tableField.setFieldType("DOUBLE");
}
NumberFormat nf = NumberFormat.getInstance();
nf.setGroupingUsed(false);
@ -2086,23 +2083,23 @@ public class DataSetTableService {
} catch (IllegalStateException e) {
String s = String.valueOf(cell.getRichStringCellValue());
if (cellType) {
tableFiled.setFieldType("TEXT");
tableFiled.setFieldSize(65533);
tableField.setFieldType("TEXT");
tableField.setFieldSize(65533);
}
return s;
}
}
if (cellTypeEnum.equals(CellType.STRING)) {
if (cellType) {
tableFiled.setFieldType("TEXT");
tableFiled.setFieldSize(65533);
tableField.setFieldType("TEXT");
tableField.setFieldSize(65533);
}
return cell.getStringCellValue();
}
if (cellTypeEnum.equals(CellType.NUMERIC)) {
if (HSSFDateUtil.isCellDateFormatted(cell)) {
if (cellType) {
tableFiled.setFieldType("DATETIME");
tableField.setFieldType("DATETIME");
}
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
try {
@ -2117,15 +2114,15 @@ public class DataSetTableService {
double eps = 1e-10;
if (value - Math.floor(value) < eps) {
if (cellType) {
if (StringUtils.isEmpty(tableFiled.getFieldType())
|| tableFiled.getFieldType().equalsIgnoreCase("TEXT")) {
tableFiled.setFieldType("LONG");
if (StringUtils.isEmpty(tableField.getFieldType())
|| tableField.getFieldType().equalsIgnoreCase("TEXT")) {
tableField.setFieldType("LONG");
}
}
return value.longValue() + "";
} else {
if (cellType) {
tableFiled.setFieldType("DOUBLE");
tableField.setFieldType("DOUBLE");
}
NumberFormat nf = NumberFormat.getInstance();
nf.setGroupingUsed(false);

114
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -1,5 +1,6 @@
package io.dataease.service.dataset;
import com.alibaba.fastjson.JSONObject;
import com.google.gson.Gson;
import io.dataease.base.domain.*;
import io.dataease.base.mapper.DatasetTableMapper;
@ -10,6 +11,7 @@ import io.dataease.commons.constants.*;
import io.dataease.commons.model.AuthURD;
import io.dataease.commons.utils.*;
import io.dataease.commons.constants.DatasourceTypes;
import io.dataease.controller.request.datasource.ApiDefinition;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.provider.ProviderFactory;
@ -63,7 +65,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.File;
import java.io.*;
import java.net.InetAddress;
import java.util.*;
import java.util.stream.Collectors;
@ -314,16 +316,13 @@ public class ExtractDataService {
try {
createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTableColumnSql);
createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTableColumnSql);
generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
generateJobFile("all_scope", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
execTime = System.currentTimeMillis();
extractData(datasetTable, "all_scope");
extractData(datasetTable, datasource, datasetTableFields, "all_scope", null);
replaceTable(DorisTableUtils.dorisName(datasetTableId));
saveSuccessLog(datasetTableTaskLog);
msg = true;
lastExecStatus = JobStatus.Completed;
} catch (Exception e) {
e.printStackTrace();
saveErrorLog(datasetTableId, taskId, e);
msg = false;
lastExecStatus = JobStatus.Error;
@ -341,6 +340,10 @@ public class ExtractDataService {
case add_scope: // 增量更新
try {
if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){
extractData(datasetTable, datasource, datasetTableFields, "incremental_add", null);
return;
}
DatasetTableIncrementalConfig datasetTableIncrementalConfig = dataSetTableService.incrementalConfig(datasetTableId);
if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) {
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, null);
@ -357,17 +360,13 @@ public class ExtractDataService {
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, datasetTable.getLastUpdateTime().toString())
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, sql);
generateJobFile("incremental_add", datasetTable, fetchSqlField(sql, datasource));
extractData(datasetTable, "incremental_add");
extractData(datasetTable, datasource, datasetTableFields, "incremental_add", sql);
}
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除
String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, datasetTable.getLastUpdateTime().toString())
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
generateTransFile("incremental_delete", datasetTable, datasource, datasetTableFields, sql);
generateJobFile("incremental_delete", datasetTable, fetchSqlField(sql, datasource));
extractData(datasetTable, "incremental_delete");
extractData(datasetTable, datasource, datasetTableFields, "incremental_delete", sql);
}
saveSuccessLog(datasetTableTaskLog);
@ -394,6 +393,97 @@ public class ExtractDataService {
}
private void extractData(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType, String selectSQL) throws Exception{
if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){
extractDataByDE(datasetTable, datasource, datasetTableFields, extractType);
return;
}
extractDataByKettle(datasetTable, datasource, datasetTableFields, extractType, selectSQL);
}
private void extractDataByDE(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType)throws Exception{
List<ApiDefinition> lists = JSONObject.parseArray(datasource.getConfiguration(), ApiDefinition.class);
lists = lists.stream().filter(item -> item.getName().equalsIgnoreCase(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable())).collect(Collectors.toList());
if(CollectionUtils.isEmpty(lists)){
throw new Exception("未找到API数据表");
}
if(lists.size() > 1 ){
throw new Exception("存在重名的API数据表");
}
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource);
datasourceRequest.setTable(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable());
Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
List<String[]> dataList = result.get("dataList");
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class);
String columns = datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")) + ",dataease_uuid";
String dataFile = null;
String script = null;
switch (extractType) {
case "all_scope":
dataFile = root_path + DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())));
break;
default:
dataFile = root_path + DorisTableUtils.dorisAddName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()));
break;
}
BufferedWriter bw = new BufferedWriter(new FileWriter(dataFile));
for (String[] strings : dataList) {
String content = "";
for (int i=0;i< strings.length;i++){
content = content + strings[i] + separator;
}
content = content + Md5Utils.md5(content);
bw.write(content);
bw.newLine();
}
bw.close();
File scriptFile = new File(root_path + datasetTable.getId() + ".sh");
scriptFile.setExecutable(true);
scriptFile.createNewFile();
BufferedWriter scriptFileBw = new BufferedWriter(new FileWriter(root_path + datasetTable.getId() + ".sh"));
scriptFileBw.write("#!/bin/sh");
scriptFileBw.newLine();
scriptFileBw.write(script);
scriptFileBw.newLine();
scriptFileBw.close();
try {
Process process = Runtime.getRuntime().exec(root_path + datasetTable.getId() + ".sh");
process.waitFor();
if(process.waitFor() != 0){
BufferedReader input = new BufferedReader(new InputStreamReader(process.getErrorStream()));
String errMsg = "";
String line = "";
while ((line = input.readLine()) != null) {
errMsg = errMsg + line + System.getProperty("line.separator");
}
throw new Exception(errMsg);
}
}catch (Exception e){
throw e;
}finally {
File deleteFile = new File(root_path + datasetTable.getId() + ".sh");
FileUtils.forceDelete(deleteFile);
}
}
private void extractDataByKettle(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType, String selectSQL)throws Exception{
generateTransFile(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
generateJobFile(extractType, datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
extractData(datasetTable, extractType);
}
private void sendWebMsg(DatasetTable datasetTable, DatasetTableTask datasetTableTask, DatasetTableTaskLog datasetTableTaskLog, Boolean status) {
String taskId = datasetTableTask.getId();
String msg = status ? "成功" : "失败";
@ -729,7 +819,7 @@ public class ExtractDataService {
datasourceRequest.setDatasource(ds);
datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> dorisFields = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed -> {
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName).forEach(filed -> {
dorisFields.add(DorisTableUtils.columnName(filed));
});
return String.join(",", dorisFields);

49
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@ -1,7 +1,9 @@
package io.dataease.service.datasource;
import cn.hutool.json.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.gson.Gson;
import com.jayway.jsonpath.JsonPath;
import io.dataease.base.domain.*;
import io.dataease.base.mapper.*;
import io.dataease.base.mapper.ext.ExtDataSourceMapper;
@ -10,12 +12,16 @@ import io.dataease.commons.exception.DEException;
import io.dataease.commons.model.AuthURD;
import io.dataease.commons.utils.AuthUtils;
import io.dataease.commons.utils.CommonThreadPool;
import io.dataease.commons.utils.HttpClientUtil;
import io.dataease.commons.utils.LogUtil;
import io.dataease.controller.ResultHolder;
import io.dataease.controller.request.DatasourceUnionRequest;
import io.dataease.controller.request.datasource.ApiDefinition;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.controller.sys.base.ConditionEntity;
import io.dataease.commons.constants.DatasourceTypes;
import io.dataease.exception.ExcelException;
import io.dataease.provider.datasource.ApiProvider;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.ProviderFactory;
import io.dataease.controller.request.datasource.DatasourceRequest;
@ -103,6 +109,9 @@ public class DatasourceService {
case ck:
datasourceDTO.setConfiguration(JSONObject.toJSONString(new Gson().fromJson(datasourceDTO.getConfiguration(), CHConfiguration.class)) );
break;
case api:
datasourceDTO.setApiConfiguration(JSONObject.parseArray(datasourceDTO.getConfiguration()));
break;
default:
break;
}
@ -260,13 +269,45 @@ public class DatasourceService {
datasources.forEach(datasource -> checkAndUpdateDatasourceStatus(datasource, true));
}
public ApiDefinition checkApiDatasource(ApiDefinition apiDefinition) throws Exception {
String response = ApiProvider.execHttpRequest(apiDefinition);
List<LinkedHashMap> datas = JsonPath.read(response,apiDefinition.getDataPath());
List<JSONObject> dataList = new ArrayList<>();
for (LinkedHashMap data : datas) {
JSONObject jsonObject = new JSONObject();
Iterator it = data.entrySet().iterator();
while (it.hasNext()){
Map.Entry entry = (Map.Entry)it.next();
jsonObject.put((String) entry.getKey(), entry.getValue());
}
dataList.add(jsonObject);
}
List<DatasetTableField> fields = new ArrayList<>();
if(CollectionUtils.isNotEmpty(dataList)){
for (Map.Entry<String, Object> stringObjectEntry : dataList.get(0).entrySet()) {
DatasetTableField tableField = new DatasetTableField();
tableField.setOriginName(stringObjectEntry.getKey());
tableField.setName(stringObjectEntry.getKey());
tableField.setSize(65535);
tableField.setDeExtractType(0);
tableField.setDeType(0);
tableField.setExtField(0);
fields.add(tableField);
}
}
apiDefinition.setDatas(dataList);
apiDefinition.setFields(fields);
return apiDefinition;
}
private void checkAndUpdateDatasourceStatus(Datasource datasource){
try {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource);
datasourceProvider.checkStatus(datasourceRequest);
datasource.setStatus("Success");
String status = datasourceProvider.checkStatus(datasourceRequest);
datasource.setStatus(status);
} catch (Exception e) {
datasource.setStatus("Error");
}
@ -277,8 +318,8 @@ public class DatasourceService {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource);
datasourceProvider.checkStatus(datasourceRequest);
datasource.setStatus("Success");
String status = datasourceProvider.checkStatus(datasourceRequest);
datasource.setStatus(status);
datasourceMapper.updateByPrimaryKeySelective(datasource);
} catch (Exception e) {
Datasource temp = datasourceMapper.selectByPrimaryKey(datasource.getId());

8
frontend/package.json
View File

@ -33,6 +33,13 @@
"jsencrypt": "^3.0.0-rc.1",
"jspdf": "^2.3.1",
"lodash": "^4.17.4",
"lodash.isboolean": "^3.0.3",
"lodash.isempty": "^4.4.0",
"lodash.isinteger": "^4.0.4",
"lodash.isnull": "^3.0.0",
"lodash.isnumber": "^3.0.3",
"lodash.isobject": "^3.0.2",
"lodash.isstring": "^4.0.1",
"normalize.css": "7.0.0",
"nprogress": "0.2.0",
"screenfull": "4.2.0",
@ -53,6 +60,7 @@
"vue-to-pdf": "^1.0.0",
"vue-uuid": "2.0.2",
"vue-video-player": "^5.0.2",
"vue2-ace-editor": "0.0.15",
"vuedraggable": "^2.24.3",
"vuex": "3.1.0",
"webpack": "^4.46.0",

8
frontend/src/api/dataset/dataset.js
View File

@ -77,6 +77,14 @@ export function listDatasource() {
})
}
export function listApiDatasource() {
return request({
url: '/datasource/list/api',
loading: true,
method: 'get'
})
}
export function getTable(id, hideMsg = false) {
return request({
url: '/dataset/table/get/' + id,

9
frontend/src/api/system/datasource.js
View File

@ -74,4 +74,13 @@ export function getSchema(data) {
})
}
export function checkApiDatasource(data){
return request({
url: 'datasource/checkApiDatasource',
method: 'post',
loading: false,
data
})
}
export default { dsGrid, addDs, editDs, delDs, validateDs, listDatasource, getSchema }

1
frontend/src/icons/svg/exclamationmark2.svg Normal file
View File

@ -0,0 +1 @@
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1629085981011" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="16882" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><defs><style type="text/css"></style></defs><path d="M637.6 82.3L610.9 643c-1.6 34.1-29.8 61-63.9 61h-70c-34.2 0-62.3-26.8-63.9-61L386.4 82.3c-1.3-27.4 20.5-50.3 48-50.3h155.3c27.4 0 49.2 22.9 47.9 50.3zM640 864c0 35.3-14.3 67.3-37.5 90.5-23.2 23.2-55.2 37.5-90.5 37.5s-67.3-14.3-90.5-37.5C398.3 931.3 384 899.3 384 864c0-70.7 57.3-128 128-128 35.3 0 67.3 14.3 90.5 37.5 23.2 23.2 37.5 55.2 37.5 90.5z" p-id="16883" fill="#FFA500"></path></svg>
Side by Side

After

Width:  |  Height:  |  Size: 770 B

31
frontend/src/lang/en.js
View File

@ -1049,6 +1049,7 @@ export default {
custom_data: 'Custom Dataset',
pls_slc_tbl_left: 'Please select the chart from the left',
add_db_table: 'Add Database Dataset',
add_api_table: 'Add API Dataset',
pls_slc_data_source: 'Please select data source',
table: 'Table',
edit: 'Edit',
@ -1298,7 +1299,35 @@ export default {
no_less_then_0: 'Parameters in advanced settings cannot be less than zero',
port_no_less_then_0: 'Port cannot be less than zero',
priority: 'Advanced setting',
extra_params: 'Extra JDBC connection string'
extra_params: 'Extra JDBC connection string',
please_input_dataPath: '请输入 JsonPath 数据路径',
warning: 'Contains invalid datasets',
data_table: 'Dataset Table',
data_table_name: 'Dataset Table name',
method: 'Request mode',
url: 'URL',
add_api_table: 'Add API table',
edit_api_table: 'Edit API table',
base_info: 'Basic information',
request: 'Request',
path_all_info: 'Please fill in the full address',
req_param: 'Request parameters',
headers: 'Request header',
key: 'Key',
value: 'Value',
data_path: 'Extract data',
data_path_desc: 'Please fill in the data path with Jsonpath',
body_form_data: 'form-data',
body_x_www_from_urlencoded: 'x-www-form-urlencoded',
body_json: 'json',
body_xml: 'xml',
body_raw: 'row',
request_body: 'Request Body',
auth_config: 'Auth config',
auth_config_info: 'Permission verification is required for the request',
verified: 'Verified',
verification_method: 'Verification Method',
username: 'Username'
},
pblink: {
key_pwd: 'Please enter the password to open the link',

31
frontend/src/lang/tw.js
View File

@ -1049,6 +1049,7 @@ export default {
custom_data: '自定義數據集',
pls_slc_tbl_left: '請從左側選擇錶',
add_db_table: '添加數據庫數據集',
add_api_table: '添加API數據集',
pls_slc_data_source: '請選擇數據源',
table: '錶',
edit: '編輯',
@ -1299,7 +1300,35 @@ export default {
no_less_then_0: '高級設置中的參數不能小於零',
port_no_less_then_0: '端口不能小於零',
priority: '高級設置',
extra_params: '額外的JDBC連接字符串'
extra_params: '額外的JDBC連接字符串',
please_input_dataPath: '請輸入 JsonPath 數據路徑',
warning: '包含無效數據機',
data_table: '數據表',
data_table_name: '數據表名稱',
method: '請求方式',
url: 'URL',
add_api_table: '添加 API 數據表',
edit_api_table: '編輯 API 數據表',
base_info: '基礎信息',
request: '請求',
path_all_info: '請輸入完整地址',
req_param: '請求參數',
headers: '請求頭',
key: '鍵',
value: '值',
data_path: '提取數據',
data_path_desc: '請用 JsonPath 填寫數據路徑',
body_form_data: 'form-data',
body_x_www_from_urlencoded: 'x-www-form-urlencoded',
body_json: 'json',
body_xml: 'xml',
body_raw: 'row',
request_body: '請求提',
auth_config: '認證配置',
auth_config_info: '請求需要進行權限校驗',
verified: '認證',
verification_method: '認證方式',
username: '用戶名'
},
pblink: {
key_pwd: '請輸入密碼打開鏈接',

31
frontend/src/lang/zh.js
View File

@ -678,6 +678,7 @@ export default {
custom_data: '自定义数据集',
pls_slc_tbl_left: '请从左侧选视图',
add_db_table: '添加数据库数据集',
add_api_table: '添加API数据集',
pls_slc_data_source: '请选择数据源',
table: '表',
edit: '编辑',
@ -1307,7 +1308,35 @@ export default {
direct: '直连模式',
extract: '抽取模式',
all_compute_mode: '直连、抽取模式',
extra_params: '额外的JDBC连接字符串'
extra_params: '额外的JDBC连接字符串',
please_input_dataPath: '请输入 JsonPath 数据路径',
warning: '包含无效数据集',
data_table: '数据表',
data_table_name: '数据表名称',
method: '请求方式',
url: 'URL',
add_api_table: '添加API数据表',
edit_api_table: '编辑API数据表',
base_info: '基础信息',
request: '请求',
path_all_info: '请填入完整地址',
req_param: '请求参数',
headers: '请求头',
key: '键',
value: '值',
data_path: '提取数据',
data_path_desc: '请用JsonPath填写数据路径',
body_form_data: 'form-data',
body_x_www_from_urlencoded: 'x-www-form-urlencoded',
body_json: 'json',
body_xml: 'xml',
body_raw: 'row',
request_body: '请求体',
auth_config: '认证配置',
auth_config_info: '请求需要进行权限校验',
verified: '认证',
verification_method: '认证方式',
username: '用户名'
},
pblink: {
key_pwd: '请输入密码打开链接',

209
frontend/src/views/dataset/add/AddApi.vue Normal file
View File

@ -0,0 +1,209 @@
<template>
<el-row style="display: flex;flex-direction: column;height: 100%">
<el-row style="height: 26px;" class="title-text">
<span style="line-height: 26px;">
{{ $t('dataset.add_api_table') }}
</span>
<el-row style="float: right">
<el-button size="mini" @click="cancel">
{{ $t('dataset.cancel') }}
</el-button>
<el-button size="mini" type="primary" :disabled="checkTableList.length < 1" @click="save">
{{ $t('dataset.confirm') }}
</el-button>
</el-row>
</el-row>
<el-divider />
<el-row>
<el-form :inline="true">
<el-form-item class="form-item">
<el-select v-model="dataSource" filterable :placeholder="$t('dataset.pls_slc_data_source')" size="mini">
<el-option
v-for="item in options"
:key="item.id"
:label="item.name"
:value="item.id"
/>
</el-select>
</el-form-item>
<el-form-item class="form-item">
<el-select v-model="mode" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
<el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.type==='es' || selectedDatasource.type==='ck' || selectedDatasource.type==='mongo' || selectedDatasource.type==='redshift' || selectedDatasource.type==='hive'" />
</el-select>
</el-form-item>
<el-form-item v-if="mode === '1'" class="form-item">
<el-select v-model="syncType" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
<el-option :label="$t('dataset.sync_now')" value="sync_now" />
<el-option :label="$t('dataset.sync_latter')" value="sync_latter" />
</el-select>
</el-form-item>
<el-form-item class="form-item" style="float: right;">
<el-input
v-model="searchTable"
size="mini"
:placeholder="$t('dataset.search')"
prefix-icon="el-icon-search"
clearable
/>
</el-form-item>
</el-form>
</el-row>
<el-col style="overflow-y: auto;">
<el-checkbox-group v-model="checkTableList" size="small">
<el-tooltip v-for="t in tableData" :key="t.name" :disabled="t.enableCheck" effect="dark" :content="$t('dataset.table_already_add_to')+': '+t.datasetPath" placement="bottom">
<el-checkbox
border
:label="t.name"
:disabled="!t.enableCheck"
>{{ showTableNameWithComment(t) }}</el-checkbox>
</el-tooltip>
</el-checkbox-group>
</el-col>
</el-row>
</template>
<script>
import { listApiDatasource, post, isKettleRunning } from '@/api/dataset/dataset'
export default {
name: 'AddApi',
props: {
param: {
type: Object,
default: null
}
},
data() {
return {
searchTable: '',
options: [],
dataSource: '',
tables: [],
checkTableList: [],
mode: '1',
syncType: 'sync_now',
tableData: [],
kettleRunning: false,
selectedDatasource: {}
}
},
watch: {
dataSource(val) {
if (val) {
post('/datasource/getTables', { id: val }).then(response => {
this.tables = response.data
this.tableData = JSON.parse(JSON.stringify(this.tables))
})
for (let i = 0; i < this.options.length; i++) {
if (this.options[i].id === val) {
this.selectedDatasource = this.options[i]
}
}
}
},
searchTable(val) {
if (val && val !== '') {
this.tableData = JSON.parse(JSON.stringify(this.tables.filter(ele => { return ele.name.toLocaleLowerCase().includes(val.toLocaleLowerCase()) })))
} else {
this.tableData = JSON.parse(JSON.stringify(this.tables))
}
}
},
mounted() {
this.initDataSource()
},
activated() {
this.initDataSource()
},
created() {
this.kettleState()
},
methods: {
initDataSource() {
listApiDatasource().then(response => {
this.options = response.data
})
},
kettleState() {
isKettleRunning().then(res => {
this.kettleRunning = res.data
})
},
showTableNameWithComment(t) {
if (t.remark) {
return `${t.name}(${t.remark})`
} else {
return `${t.name}`
}
},
save() {
let ds = {}
this.options.forEach(ele => {
if (ele.id === this.dataSource) {
ds = ele
}
})
const sceneId = this.param.id
const dataSourceId = this.dataSource
const tables = []
const mode = this.mode
const syncType = this.syncType
this.checkTableList.forEach(function(name) {
tables.push({
name: ds.name + '_' + name,
sceneId: sceneId,
dataSourceId: dataSourceId,
type: 'api',
syncType: syncType,
mode: parseInt(mode),
info: JSON.stringify({ table: name })
})
})
post('/dataset/table/batchAdd', tables).then(response => {
this.$emit('saveSuccess', tables[0])
this.cancel()
})
},
cancel() {
this.dataReset()
this.$emit('switchComponent', { name: '' })
},
dataReset() {
this.searchTable = ''
this.options = []
this.dataSource = ''
this.tables = []
this.checkTableList = []
}
}
}
</script>
<style scoped>
.el-divider--horizontal {
margin: 12px 0;
}
.form-item {
margin-bottom: 6px;
}
.el-checkbox {
margin-bottom: 14px;
margin-left: 0;
margin-right: 14px;
}
.el-checkbox.is-bordered + .el-checkbox.is-bordered {
margin-left: 0;
}
span{
font-size: 14px;
}
</style>

6
frontend/src/views/dataset/data/UpdateInfo.vue
View File

@ -263,7 +263,7 @@
<el-divider />
<el-row style="height: 26px;">
<el-row style="height: 26px;" v-if="table.type !== 'api'">
<el-row>
<el-col :span="4"><span>{{ $t('dataset.incremental_update_type') }}:</span></el-col>
<el-col :span="18">
@ -275,7 +275,7 @@
</el-row>
</el-row>
<el-row style="height: 26px;">
<el-row style="height: 26px;" v-if="table.type !== 'api'">
<el-row>
<el-col :span="4" style="height: 26px;"><span style="display: inline-block;height: 26px;line-height: 26px;">{{ $t('dataset.param') }}:</span></el-col>
<el-col :span="18">
@ -285,7 +285,7 @@
</el-row>
</el-row>
<el-row>
<el-row v-if="table.type !== 'api'">
<el-col style="min-width: 200px;">
<codemirror
ref="myCm"

2
frontend/src/views/dataset/data/ViewTable.vue
View File

@ -52,7 +52,7 @@
<el-tab-pane v-if="!hideCustomDs && table.type !== 'union' && table.type !== 'custom' && !(table.type === 'sql' && table.mode === 0)" :label="$t('dataset.join_view')" name="joinView">
<union-view :param="param" :table="table" />
</el-tab-pane>
<el-tab-pane v-if="table.mode === 1 && (table.type === 'excel' || table.type === 'db' || table.type === 'sql')" :label="$t('dataset.update_info')" name="updateInfo">
<el-tab-pane v-if="table.mode === 1 && (table.type === 'excel' || table.type === 'db' || table.type === 'sql' || table.type === 'api')" :label="$t('dataset.update_info')" name="updateInfo">
<update-info v-if="tabActive=='updateInfo'" :param="param" :table="table" />
</el-tab-pane>
<el-tab-pane v-if="isPluginLoaded && hasDataPermission('manage',param.privileges)" :lazy="true" :label="$t('dataset.row_permissions')" name="rowPermissions">

7
frontend/src/views/dataset/group/Group.vue
View File

@ -99,6 +99,10 @@
<svg-icon icon-class="ds-union" class="ds-icon-union" />
{{ $t('dataset.union_data') }}
</el-dropdown-item>
<el-dropdown-item :command="beforeClickAddData('api',data)">
<svg-icon icon-class="ds-union" class="ds-icon-union" />
{{ $t('dataset.union_data') }}
</el-dropdown-item>
</el-dropdown-menu>
</el-dropdown>
</el-dropdown-item>
@ -562,6 +566,9 @@ export default {
case 'union':
this.addData('AddUnion')
break
case 'api':
this.addData('AddApi')
break
}
},

6
frontend/src/views/dataset/index.vue
View File

@ -20,6 +20,7 @@ import Group from './group/Group'
import DataHome from './data/DataHome'
import ViewTable from './data/ViewTable'
import AddDB from './add/AddDB'
import AddApi from './add/AddApi'
import AddSQL from './add/AddSQL'
import AddExcel from './add/AddExcel'
import AddCustom from './add/AddCustom'
@ -29,7 +30,7 @@ import { removeClass } from '@/utils'
import { checkCustomDs } from '@/api/dataset/dataset'
export default {
name: 'DataSet',
components: { DeMainContainer, DeContainer, DeAsideContainer, Group, DataHome, ViewTable, AddDB, AddSQL, AddExcel, AddCustom },
components: { DeMainContainer, DeContainer, DeAsideContainer, Group, DataHome, ViewTable, AddDB, AddSQL, AddExcel, AddCustom, AddApi},
data() {
return {
component: DataHome,
@ -76,6 +77,9 @@ export default {
case 'FieldEdit':
this.component = FieldEdit
break
case 'AddApi':
this.component = AddApi
break
default:
this.component = DataHome
break

91
frontend/src/views/system/datasource/ApiAuthConfig.vue Normal file
View File

@ -0,0 +1,91 @@
<template xmlns:v-slot="http://www.w3.org/1999/XSL/Transform">
<el-tabs v-model="activeName">
<!-- 认证-->
<el-tab-pane :label="$t('datasource.verified')" name="verified">
<el-form :model="authConfig" :rules="rule" ref="authConfig" label-position="right">
<el-form-item :label="$t('datasource.verification_method')" prop="verification">
<el-select v-model="authConfig.verification" @change="change"
:placeholder="$t('datasource.verification_method')" filterable size="small">
<el-option
v-for="item in options"
:key="item.name"
:label="item.name"
:value="item.name">
</el-option>
</el-select>
</el-form-item>
<el-form-item :label="$t('datasource.username')" prop="username"
v-if="authConfig.verification!=undefined && authConfig.verification !='No Auth'">
<el-input :placeholder="$t('datasource.username')" v-model="authConfig.username"
class="ms-http-input" size="small">
</el-input>
</el-form-item>
<el-form-item :label="$t('datasource.password')" prop="password"
v-if=" authConfig.verification!=undefined && authConfig.verification !='No Auth'">
<el-input v-model="authConfig.password" :placeholder="$t('datasource.password')" show-password autocomplete="off"
maxlength="50" show-word-limit/>
</el-form-item>
</el-form>
</el-tab-pane>
</el-tabs>
</template>
<script>
export default {
name: "ApiAuthConfig",
components: {},
props: {
request: {},
encryptShow: {
type: Boolean,
default: true,
}
},
watch: {
request() {
this.initData();
}
},
created() {
this.initData();
},
data() {
return {
options: [{name: "No Auth"}, {name: "Basic Auth"}],
encryptOptions: [{id: false, name: this.$t('commons.encrypted')}],
activeName: "verified",
rule: {},
authConfig: {}
}
},
methods: {
change() {
if (this.authConfig.verification === "Basic Auth") {
this.authConfig.verification = "Basic Auth";
this.request.authManager = this.authConfig;
} else {
this.authConfig.verification = "No Auth";
this.request.authManager = this.authConfig;
}
},
initData() {
if (this.request.authManager) {
this.authConfig = this.request.authManager;
}
}
}
}
</script>
<style scoped>
/deep/ .el-tabs__nav-wrap::after {
height: 0px;
}
</style>

331
frontend/src/views/system/datasource/ApiBody.vue Normal file
View File

@ -0,0 +1,331 @@
<template>
<div>
<el-radio-group v-model="body.type" size="mini">
<el-radio :disabled="isReadOnly" :label="type.FORM_DATA" @change="modeChange">
{{ $t('datasource.body_form_data') }}
</el-radio>
<el-radio :disabled="isReadOnly" :label="type.WWW_FORM" @change="modeChange">
{{ $t('datasource.body_x_www_from_urlencoded') }}
</el-radio>
<el-radio :disabled="isReadOnly" :label="type.JSON" @change="modeChange">
{{ $t('datasource.body_json') }}
</el-radio>
<el-radio :disabled="isReadOnly" :label="type.XML" @change="modeChange">
{{ $t('datasource.body_xml') }}
</el-radio>
<el-radio :disabled="isReadOnly" :label="type.RAW" @change="modeChange">
{{ $t('datasource.body_raw') }}
</el-radio>
</el-radio-group>
<div style="min-width: 1200px;" v-if="body.type == 'Form_Data' || body.type == 'WWW_FORM'">
<api-variable
:with-mor-setting="true"
:is-read-only="isReadOnly"
:parameters="body.kvs"
:isShowEnable="isShowEnable"
type="body"/>
</div>
<div v-if="body.type == 'JSON'">
<code-edit
:read-only="isReadOnly"
:data.sync="body.raw"
:modes="modes"
:mode="'json'"
height="400px"
ref="codeEdit"/>
</div>
<div class="ms-body" v-if="body.type == 'XML'">
<code-edit
:read-only="isReadOnly"
:data.sync="body.raw"
:modes="modes"
:mode="'text'"
ref="codeEdit"/>
</div>
<div class="ms-body" v-if="body.type == 'Raw'">
<code-edit
:read-only="isReadOnly"
:data.sync="body.raw"
:modes="modes"
ref="codeEdit"/>
</div>
</div>
</template>
<script>
import ApiKeyValue from "./ApiKeyValue";
import {BODY_TYPE, KeyValue} from "./ApiTestModel";
import CodeEdit from "./CodeEdit";
import ApiVariable from "./ApiVariable";
import Convert from "./convert";
export default {
name: "ApiBody",
components: {
ApiVariable,
CodeEdit,
ApiKeyValue
},
props: {
body: {},
headers: Array,
isReadOnly: {
type: Boolean,
default: false
},
isShowEnable: {
type: Boolean,
default: false
}
},
data() {
return {
type: BODY_TYPE,
modes: ['text', 'json', 'xml', 'html'],
jsonSchema: "JSON",
codeEditActive: true,
hasOwnProperty: Object.prototype.hasOwnProperty,
propIsEnumerable: Object.prototype.propertyIsEnumerable
};
},
watch: {
'body.typeChange'() {
this.reloadCodeEdit();
},
'body.raw'() {
if (this.body.format !== 'JSON-SCHEMA' && this.body.raw) {
try {
const MsConvert = new Convert();
let data = MsConvert.format(JSON.parse(this.body.raw));
if (this.body.jsonSchema) {
this.body.jsonSchema = this.deepAssign(data);
} else {
this.body.jsonSchema = data;
}
} catch (ex) {
this.body.jsonSchema = "";
}
}
},
},
methods: {
isObj(x) {
let type = typeof x;
return x !== null && (type === 'object' || type === 'function');
},
toObject(val) {
if (val === null || val === undefined) {
return;
}
return Object(val);
},
assignKey(to, from, key) {
let val = from[key];
if (val === undefined || val === null) {
return;
}
if (!this.hasOwnProperty.call(to, key) || !this.isObj(val)) {
to[key] = val;
} else {
to[key] = this.assign(Object(to[key]), from[key]);
}
},
assign(to, from) {
if (to === from) {
return to;
}
from = Object(from);
for (let key in from) {
if (this.hasOwnProperty.call(from, key)) {
this.assignKey(to, from, key);
}
}
if (Object.getOwnPropertySymbols) {
let symbols = Object.getOwnPropertySymbols(from);
for (let i = 0; i < symbols.length; i++) {
if (this.propIsEnumerable.call(from, symbols[i])) {
this.assignKey(to, from, symbols[i]);
}
}
}
return to;
},
deepAssign(target) {
target = this.toObject(target);
for (let s = 1; s < arguments.length; s++) {
this.assign(target, arguments[s]);
}
return target;
},
reloadCodeEdit() {
this.codeEditActive = false;
this.$nextTick(() => {
this.codeEditActive = true;
});
},
formatChange() {
const MsConvert = new Convert();
if (this.body.format === 'JSON-SCHEMA') {
if (this.body.raw && !this.body.jsonSchema) {
this.body.jsonSchema = MsConvert.format(JSON.parse(this.body.raw));
}
} else {
if (this.body.jsonSchema) {
MsConvert.schemaToJsonStr(this.body.jsonSchema, (result) => {
this.$set(this.body, 'raw', result);
this.reloadCodeEdit();
});
}
}
},
modeChange(mode) {
switch (this.body.type) {
case "JSON":
this.setContentType("application/json");
break;
case "XML":
this.setContentType("text/xml");
break;
case "WWW_FORM":
this.setContentType("application/x-www-form-urlencoded");
break;
// todo from data
case "BINARY":
this.setContentType("application/octet-stream");
break;
default:
this.removeContentType();
break;
}
},
setContentType(value) {
let isType = false;
this.headers.forEach(item => {
if (item.name === "Content-Type") {
item.value = value;
isType = true;
}
})
if (!isType) {
this.headers.unshift(new KeyValue({name: "Content-Type", value: value}));
this.$emit('headersChange');
}
},
removeContentType() {
for (let index in this.headers) {
if (this.headers[index].name === "Content-Type") {
this.headers.splice(index, 1);
this.$emit('headersChange');
return;
}
}
},
batchAdd() {
this.$refs.batchAddParameter.open();
},
format(array, obj) {
if (array) {
let isAdd = true;
for (let i in array) {
let item = array[i];
if (item.name === obj.name) {
item.value = obj.value;
isAdd = false;
}
}
if (isAdd) {
this.body.kvs.unshift(obj);
}
}
},
batchSave(data) {
if (data) {
let params = data.split("\n");
let keyValues = [];
params.forEach(item => {
let line = [];
line[0] = item.substring(0, item.indexOf(":"));
line[1] = item.substring(item.indexOf(":") + 1, item.length);
let required = false;
keyValues.unshift(new KeyValue({
name: line[0],
required: required,
value: line[1],
description: line[2],
type: "text",
valid: false,
file: false,
encode: true,
enable: true,
contentType: "text/plain"
}));
})
keyValues.forEach(item => {
this.format(this.body.kvs, item);
})
}
},
},
created() {
if (!this.body.type) {
this.body.type = BODY_TYPE.FORM_DATA;
}
if (this.body.kvs) {
this.body.kvs.forEach(param => {
if (!param.type) {
param.type = 'text';
}
});
}
}
}
</script>
<style scoped>
.textarea {
margin-top: 10px;
}
.ms-body {
padding: 15px 0;
height: 400px;
}
.el-dropdown {
margin-left: 20px;
line-height: 30px;
}
.ace_editor {
border-radius: 5px;
}
.el-radio-group {
margin: 10px 10px;
margin-top: 15px;
}
.ms-el-link {
float: right;
margin-right: 45px;
}
</style>

394
frontend/src/views/system/datasource/ApiHttpRequestForm.vue Normal file
View File

@ -0,0 +1,394 @@
<template xmlns:v-slot="http://www.w3.org/1999/XSL/Transform">
<div>
<el-row>
<el-col :span="spanCount">
<!-- HTTP 请求参数 -->
<div style="border:1px #DCDFE6 solid; height: 100%;border-radius: 4px ;width: 100%">
<el-tabs v-model="activeName" class="request-tabs">
<!-- 请求头-->
<el-tab-pane :label="$t('datasource.headers')" name="headers">
<el-tooltip class="item-tabs" effect="dark" :content="$t('datasource.headers')" placement="top-start" slot="label">
<span>{{ $t('datasource.headers') }}
<div class="el-step__icon is-text ms-api-col ms-header" v-if="headers.length>1">
<div class="el-step__icon-inner">{{ headers.length - 1 }}</div>
</div>
</span>
</el-tooltip>
<api-key-value :show-desc="true" :suggestions="headerSuggestions" :items="headers"/>
</el-tab-pane>
<!--请求体-->
<el-tab-pane v-if="isBodyShow" :label="$t('datasource.request_body')" name="body" style="overflow: auto">
<api-body @headersChange="reloadBody" :isShowEnable="isShowEnable" :headers="headers" :body="request.body"/>
</el-tab-pane>
<!-- 认证配置 -->
<el-tab-pane :label="$t('datasource.auth_config')" name="authConfig">
<el-tooltip class="item-tabs" effect="dark" :content="$t('datasource.auth_config_info')" placement="top-start" slot="label">
<span>{{ $t('datasource.auth_config') }}</span>
</el-tooltip>
<api-auth-config :request="request"/>
</el-tab-pane>
</el-tabs>
</div>
</el-col>
</el-row>
</div>
</template>
<script>
import ApiKeyValue from "@/views/system/datasource/ApiKeyValue";
import ApiBody from "@/views/system/datasource/ApiBody";
import ApiAuthConfig from "@/views/system/datasource/ApiAuthConfig";
import {Body, KeyValue} from "@/views/system/datasource/ApiTestModel";
import Convert from "@/views/system/datasource/convert";
export default {
name: "ApiHttpRequestForm",
components: {
ApiAuthConfig,
ApiBody,
ApiKeyValue
},
props: {
method: String,
request: {},
response: {},
definitionTest: {
type: Boolean,
default() {
return false;
}
},
showScript: {
type: Boolean,
default: true,
},
headers: {
type: Array,
default() {
return [];
}
},
referenced: {
type: Boolean,
default: false,
},
isShowEnable: {
type: Boolean,
default: false,
},
jsonPathList: Array,
isReadOnly: {
type: Boolean,
default: false
},
type: String,
},
data() {
let validateURL = (rule, value, callback) => {
try {
new URL(this.addProtocol(this.request.url));
} catch (e) {
callback(this.$t('api_test.request.url_invalid'));
}
};
return {
activeName: 'headers',
rules: {
name: [
{max: 300, message: this.$t('commons.input_limit', [1, 300]), trigger: 'blur'}
],
url: [
{max: 500, required: true, message: this.$t('commons.input_limit', [1, 500]), trigger: 'blur'},
{validator: validateURL, trigger: 'blur'}
],
path: [
{max: 500, message: this.$t('commons.input_limit', [0, 500]), trigger: 'blur'},
]
},
spanCount: 21,
isReloadData: false,
isBodyShow: true,
dialogVisible: false,
hasOwnProperty: Object.prototype.hasOwnProperty,
propIsEnumerable: Object.prototype.propertyIsEnumerable,
headerSuggestions: [
{value: 'Accept'},
{value: 'Accept-Charset'},
{value: 'Accept-Language'},
{value: 'Accept-Datetime'},
{value: 'Authorization'},
{value: 'Cache-Control'},
{value: 'Connection'},
{value: 'Cookie'},
{value: 'Content-Length'},
{value: 'Content-MD5'},
{value: 'Content-Type'},
{value: 'Date'},
{value: 'Expect'},
{value: 'From'},
{value: 'Host'},
{value: 'If-Match'},
{value: 'If-Modified-Since'},
{value: 'If-None-Match'},
{value: 'If-Range'},
{value: 'If-Unmodified-Since'},
{value: 'Max-Forwards'},
{value: 'Origin'},
{value: 'Pragma'},
{value: 'Proxy-Authorization'},
{value: 'Range'},
{value: 'Referer'},
{value: 'TE'},
{value: 'User-Agent'},
{value: 'Upgrade'},
{value: 'Via'},
{value: 'Warning'}
]
}
},
created() {
if (!this.referenced && this.showScript) {
this.spanCount = 21;
} else {
this.spanCount = 24;
}
this.init();
},
watch: {
'request.changeId'() {
if (this.request.headers && this.request.headers.length > 1) {
this.activeName = 'headers';
}
if (this.request.rest && this.request.rest.length > 1) {
this.activeName = 'rest';
}
if (this.request.arguments && this.request.arguments.length > 1) {
this.activeName = 'parameters';
}
if(this.request.body) {
this.request.body.typeChange = this.request.changeId;
}
this.reload();
}
},
methods: {
generate() {
if (this.request.body && (this.request.body.jsonSchema || this.request.body.raw)) {
if (!this.request.body.jsonSchema) {
const MsConvert = new Convert();
this.request.body.jsonSchema = MsConvert.format(JSON.parse(this.request.body.raw));
}
this.$post('/api/test/data/generator', this.request.body.jsonSchema, response => {
if (response.data) {
if (this.request.body.format !== 'JSON-SCHEMA') {
this.request.body.raw = response.data;
} else {
const MsConvert = new Convert();
let data = MsConvert.format(JSON.parse(response.data));
this.request.body.jsonSchema = this.deepAssign(this.request.body.jsonSchema, data);
}
this.reloadBody();
}
});
}
},
remove(row) {
let index = this.request.hashTree.indexOf(row);
this.request.hashTree.splice(index, 1);
this.reload();
},
copyRow(row) {
let obj = JSON.parse(JSON.stringify(row));
obj.id = getUUID();
this.request.hashTree.push(obj);
this.reload();
},
reload() {
this.isReloadData = true
this.$nextTick(() => {
this.isReloadData = false
})
},
init() {
if (Object.prototype.toString.call(this.request).match(/\[object (\w+)\]/)[1].toLowerCase() !== 'object') {
this.request = JSON.parse(this.request);
}
if (!this.request.body) {
this.request.body = new Body();
}
if (!this.request.body.kvs) {
this.request.body.kvs = [];
}
if (!this.request.rest) {
this.request.rest = [];
}
if (!this.request.arguments) {
this.request.arguments = [];
}
},
reloadBody() {
// 解决修改请求头后 body 显示错位
this.isBodyShow = false;
this.$nextTick(() => {
this.isBodyShow = true;
});
},
batchAdd() {
this.$refs.batchAddParameter.open();
},
format(array, obj) {
if (array) {
let isAdd = true;
for (let i in array) {
let item = array[i];
if (item.name === obj.name) {
item.value = obj.value;
isAdd = false;
}
}
if (isAdd) {
switch (this.activeName) {
case "parameters":
this.request.arguments.unshift(obj);
break;
case "rest":
this.request.rest.unshift(obj);
break;
case "headers":
this.request.headers.unshift(obj);
break;
default:
break;
}
}
}
},
batchSave(data) {
if (data) {
let params = data.split("\n");
let keyValues = [];
params.forEach(item => {
let line = item.split(/：|:/);
let required = false;
keyValues.unshift(new KeyValue({
name: line[0],
required: !required,
value: line[1],
description: line[2],
type: "text",
valid: false,
file: false,
encode: true,
enable: true,
contentType: "text/plain"
}));
})
keyValues.forEach(item => {
switch (this.activeName) {
case "parameters":
this.format(this.request.arguments, item);
break;
case "rest":
this.format(this.request.rest, item);
break;
case "headers":
this.format(this.request.headers, item);
break;
default:
break;
}
})
}
},
isObj(x) {
let type = typeof x;
return x !== null && (type === 'object' || type === 'function');
},
toObject(val) {
if (val === null || val === undefined) {
return;
}
return Object(val);
},
assignKey(to, from, key) {
let val = from[key];
if (val === undefined || val === null) {
return;
}
if (!this.hasOwnProperty.call(to, key) || !this.isObj(val)) {
to[key] = val;
} else {
to[key] = this.assign(Object(to[key]), from[key]);
}
},
assign(to, from) {
if (to === from) {
return to;
}
from = Object(from);
for (let key in from) {
if (this.hasOwnProperty.call(from, key)) {
this.assignKey(to, from, key);
}
}
if (Object.getOwnPropertySymbols) {
let symbols = Object.getOwnPropertySymbols(from);
for (let i = 0; i < symbols.length; i++) {
if (this.propIsEnumerable.call(from, symbols[i])) {
this.assignKey(to, from, symbols[i]);
}
}
}
return to;
},
deepAssign(target) {
target = this.toObject(target);
for (let s = 1; s < arguments.length; s++) {
this.assign(target, arguments[s]);
}
return target;
}
}
}
</script>
<style scoped>
.ms-query {
background: #783887;
color: white;
height: 18px;
border-radius: 42%;
}
.ms-header {
background: #783887;
color: white;
height: 18px;
border-radius: 42%;
}
.request-tabs {
margin: 20px;
min-height: 200px;
}
.ms-el-link {
float: right;
margin-right: 45px;
}
</style>

218
frontend/src/views/system/datasource/ApiKeyValue.vue Normal file
View File

@ -0,0 +1,218 @@
<template>
<div v-loading="loading">
<div class="kv-row item" v-for="(item, index) in items" :key="index">
<el-row type="flex" :gutter="20" justify="space-between" align="middle">
<i class="el-icon-top" style="cursor:pointer" @click="moveTop(index)"/>
<i class="el-icon-bottom" style="cursor:pointer;" @click="moveBottom(index)"/>
<el-col class="item" v-if="unShowSelect===false">
<el-input v-if="!suggestions" :disabled="isReadOnly" v-model="item.name" size="small" maxlength="200"
@change="change"
:placeholder="keyText" show-word-limit/>
<el-autocomplete :disabled="isReadOnly" :maxlength="400" v-if="suggestions" v-model="item.name" size="small"
:fetch-suggestions="querySearch" @change="change" :placeholder="keyText"
show-word-limit/>
</el-col>
<el-col class="item" v-if="unShowSelect===true">
<el-input v-if="suggestions" :disabled="isReadOnly" v-model="item.name" size="small" maxlength="200" :placeholder="keyText" show-word-limit/>
</el-col>
<el-col class="item">
<el-input v-if="!needMock" :disabled="isReadOnly" v-model="item.value" size="small" @change="change"
:placeholder="unShowSelect?$t('commons.description'):valueText" show-word-limit/>
</el-col>
<el-col class="item" v-if="showDesc">
<el-input v-model="item.description" size="small" maxlength="200" :placeholder="$t('commons.description')" show-word-limit></el-input>
</el-col>
<el-col class="item kv-delete">
<el-button size="mini" class="el-icon-delete-solid" circle @click="remove(index)" :disabled="isDisable(index)"/>
</el-col>
</el-row>
</div>
</div>
</template>
<script>
import {KeyValue} from "./ApiTestModel";
import Vue from 'vue';
export default {
name: "ApiKeyValue",
components: {},
props: {
keyPlaceholder: String,
valuePlaceholder: String,
isShowEnable: {
type: Boolean,
default: false
},
unShowSelect:{
type: Boolean,
default: false
},
description: String,
items: Array,
isReadOnly: {
type: Boolean,
default: false
},
suggestions: Array,
needMock: {
type: Boolean,
default: false
},
showDesc: Boolean,
appendToBody: {
type: Boolean,
default() {
return false;
}
},
},
data() {
return {
keyValues: [],
loading: false,
currentItem: {},
isSelectAll: true
}
},
computed: {
keyText() {
return this.keyPlaceholder || this.$t("datasource.key");
},
valueText() {
return this.valuePlaceholder || this.$t("datasource.value");
}
},
watch: {
isSelectAll: function (to, from) {
if (from == false && to == true) {
this.selectAll();
} else if (from == true && to == false) {
this.invertSelect();
}
}
},
methods: {
advanced(item) {
this.currentItem = item;
this.$refs.variableAdvance.open();
},
funcFilter(queryString) {
return (func) => {
return (func.name.toLowerCase().indexOf(queryString.toLowerCase()) > -1);
};
},
moveBottom(index) {
if (this.items.length < 2 || index === this.items.length - 2) {
return;
}
let thisRow = this.items[index];
let nextRow = this.items[index + 1];
Vue.set(this.items, index + 1, thisRow);
Vue.set(this.items, index, nextRow)
},
moveTop(index) {
if (index === 0) {
return;
}
let thisRow = this.items[index];
let lastRow = this.items[index - 1];
Vue.set(this.items, index - 1, thisRow);
Vue.set(this.items, index, lastRow)
},
reload() {
this.loading = true
this.$nextTick(() => {
this.loading = false
})
},
remove: function (index) {
// 移除整行输入控件及内容
this.items.splice(index, 1);
this.$emit('change', this.items);
},
change: function () {
let isNeedCreate = true;
let removeIndex = -1;
this.items.forEach((item, index) => {
if (!item.name && !item.value) {
// 多余的空行
if (index !== this.items.length - 1) {
removeIndex = index;
}
// 没有空行，需要创建空行
isNeedCreate = false;
}
});
if (isNeedCreate) {
this.items.push(new KeyValue({enable: true}));
}
this.$emit('change', this.items);
// TODO 检查key重复
},
isDisable: function (index) {
return this.items.length - 1 === index;
},
querySearch(queryString, cb) {
let suggestions = this.suggestions;
let results = queryString ? suggestions.filter(this.createFilter(queryString)) : suggestions;
cb(results);
},
createFilter(queryString) {
return (restaurant) => {
return (restaurant.value.toLowerCase().indexOf(queryString.toLowerCase()) === 0);
};
},
selectAll() {
this.items.forEach(item => {
item.enable = true;
});
},
invertSelect() {
this.items.forEach(item => {
item.enable = false;
});
},
},
created() {
if (this.items.length === 0 || this.items[this.items.length - 1].name) {
this.items.push(new KeyValue({enable: true, name: '', value: ''}));
}
}
}
</script>
<style scoped>
.kv-description {
font-size: 13px;
}
.kv-row {
margin-top: 10px;
}
.kv-checkbox {
width: 20px;
margin-right: 10px;
}
.kv-delete {
width: 60px;
}
.el-autocomplete {
width: 100%;
}
i:hover {
color: #783887;
}
</style>

165
frontend/src/views/system/datasource/ApiTestModel.js Normal file
View File

@ -0,0 +1,165 @@
export class BaseConfig {
set(options, notUndefined) {
options = this.initOptions(options)
for (let name in options) {
if (options.hasOwnProperty(name)) {
if (!(this[name] instanceof Array)) {
if (notUndefined === true) {
this[name] = options[name] === undefined ? this[name] : options[name];
} else {
this[name] = options[name];
}
}
}
}
}
sets(types, options) {
options = this.initOptions(options)
if (types) {
for (let name in types) {
if (types.hasOwnProperty(name) && options.hasOwnProperty(name)) {
options[name].forEach(o => {
this[name].push(new types[name](o));
})
}
}
}
}
initOptions(options) {
return options || {};
}
isValid() {
return true;
}
}
export class KeyValue extends BaseConfig {
constructor(options) {
options = options || {};
options.enable = options.enable === undefined ? true : options.enable;
super();
this.name = undefined;
this.value = undefined;
this.type = undefined;
this.files = undefined;
this.enable = undefined;
this.uuid = undefined;
this.time = undefined;
this.contentType = undefined;
this.set(options);
}
isValid() {
return (!!this.name || !!this.value) && this.type !== 'file';
}
isFile() {
return (!!this.name || !!this.value) && this.type === 'file';
}
}
export class Body extends BaseConfig {
constructor(options) {
super();
this.type = "KeyValue";
this.raw = undefined;
this.kvs = [];
this.binary = [];
this.set(options);
this.sets({kvs: KeyValue}, {binary: KeyValue}, options);
}
isValid() {
if (this.isKV()) {
return this.kvs.some(kv => {
return kv.isValid();
})
} else {
return !!this.raw;
}
}
isKV() {
return [BODY_TYPE.FORM_DATA, BODY_TYPE.WWW_FORM, BODY_TYPE.BINARY].indexOf(this.type) > 0;
}
}
export const createComponent = function (name) {
let component = MODELS[name];
if (component) {
return new component();
} else {
return new UnsupportedComponent()
}
}
export const BODY_TYPE = {
KV: "KeyValue",
FORM_DATA: "Form_Data",
RAW: "Raw",
WWW_FORM: "WWW_FORM",
XML: "XML",
JSON: "JSON"
}
export class Scenario extends BaseConfig {
constructor(options = {}) {
super();
this.id = undefined;
this.name = undefined;
this.url = undefined;
this.variables = [];
this.headers = [];
this.requests = [];
this.environmentId = undefined;
this.dubboConfig = undefined;
this.environment = undefined;
this.enableCookieShare = false;
this.enable = true;
this.databaseConfigs = [];
this.tcpConfig = undefined;
this.set(options);
this.sets({
variables: KeyValue,
headers: KeyValue,
requests: RequestFactory,
databaseConfigs: DatabaseConfig
}, options);
}
initOptions(options = {}) {
options.id = options.id || uuid();
options.requests = options.requests || [new RequestFactory()];
options.databaseConfigs = options.databaseConfigs || [];
options.dubboConfig = new DubboConfig(options.dubboConfig);
options.tcpConfig = new TCPConfig(options.tcpConfig);
return options;
}
clone() {
let clone = new Scenario(this);
clone.id = uuid();
return clone;
}
isValid() {
if (this.enable) {
for (let i = 0; i < this.requests.length; i++) {
let validator = this.requests[i].isValid(this.environmentId, this.environment);
if (!validator.isValid) {
return validator;
}
}
}
return {isValid: true};
}
isReference() {
return this.id.indexOf("#") !== -1
}
}

314
frontend/src/views/system/datasource/ApiVariable.vue Normal file
View File

@ -0,0 +1,314 @@
<template>
<div style="margin-bottom: 20px">
<span class="kv-description" v-if="description">
{{ description }}
</span>
<div class="item kv-row" v-for="(item, index) in parameters" :key="index">
<el-row type="flex" :gutter="20" justify="space-between" align="middle">
<span style="margin-left: 10px"></span>
<i class="el-icon-top" style="cursor:pointer" @click="moveTop(index)"/>
<i class="el-icon-bottom" style="cursor:pointer;" @click="moveBottom(index)"/>
<el-col class="item">
<el-input v-if="!suggestions" :disabled="isReadOnly" v-model="item.name" size="small" maxlength="200"
@change="change" :placeholder="keyText" show-word-limit>
<template v-slot:prepend>
<el-select v-if="type === 'body'" :disabled="isReadOnly" class="kv-type" v-model="item.type"
@change="typeChange(item)">
<el-option value="text"/>
<el-option value="json"/>
</el-select>
</template>
</el-input>
<el-autocomplete :disabled="isReadOnly" v-if="suggestions" v-model="item.name" size="small"
:fetch-suggestions="querySearch" @change="change" :placeholder="keyText" show-word-limit/>
</el-col>
<el-col class="item" v-if="isActive && item.type !== 'file'">
<el-input :disabled="isReadOnly"
size="small"
class="input-with-autocomplete"
v-model="item.value"
:placeholder="valueText"
value-key="name"
highlight-first-item
@select="change">
</el-input>
<!-- <el-autocomplete-->
<!-- :disabled="isReadOnly"-->
<!-- size="small"-->
<!-- class="input-with-autocomplete"-->
<!-- v-model="item.value"-->
<!-- placeholder="valueText"-->
<!-- value-key="name"-->
<!-- highlight-first-item-->
<!-- @select="change">-->
<!-- </el-autocomplete>-->
</el-col>
<el-col class="item">
<el-input v-model="item.description" size="small" maxlength="200"
:placeholder="$t('commons.description')" show-word-limit>
</el-input>
<el-autocomplete :disabled="isReadOnly" v-if="suggestions" v-model="item.name" size="small"
:fetch-suggestions="querySearch" @change="change" :placeholder="keyText" show-word-limit/>
</el-col>
<el-col v-if="type === 'body'" class="item kv-select">
<el-input :disabled="isReadOnly" v-model="item.contentType" size="small"
@change="change" :placeholder="$t('api_test.request.content_type')" show-word-limit>
</el-input>
</el-col>
<el-col class="item kv-delete">
<el-button size="mini" class="el-icon-delete-solid" circle @click="remove(index)"
:disabled="isDisable(index) || isReadOnly"/>
</el-col>
</el-row>
</div>
</div>
</template>
<script>
import {KeyValue, Scenario} from "./ApiTestModel";
import Vue from 'vue';
export default {
name: "ApiVariable",
components: {},
props: {
keyPlaceholder: String,
valuePlaceholder: String,
description: String,
parameters: Array,
rest: Array,
environment: Object,
scenario: Scenario,
type: {
type: String,
default: ''
},
appendDialogToBody: {
type: Boolean,
default() {
return false;
}
},
isReadOnly: {
type: Boolean,
default: false
},
isShowEnable: {
type: Boolean,
default: true
},
suggestions: Array,
withMorSetting: Boolean
},
data() {
return {
currentItem: null,
requireds: [
{name: this.$t('commons.selector.required'), id: true},
{name: this.$t('commons.selector.not_required'), id: false}
],
isSelectAll: true,
isActive: true,
}
},
watch: {
isSelectAll: function (to, from) {
if (from == false && to == true) {
this.selectAll();
} else if (from == true && to == false) {
this.invertSelect();
}
},
},
computed: {
keyText() {
return this.keyPlaceholder || this.$t("datasource.key");
},
valueText() {
return this.valuePlaceholder || this.$t("datasource.value");
}
},
methods: {
moveBottom(index) {
if (this.parameters.length < 2 || index === this.parameters.length - 2) {
return;
}
let thisRow = this.parameters[index];
let nextRow = this.parameters[index + 1];
Vue.set(this.parameters, index + 1, thisRow);
Vue.set(this.parameters, index, nextRow)
},
moveTop(index) {
if (index === 0) {
return;
}
let thisRow = this.parameters[index];
let lastRow = this.parameters[index - 1];
Vue.set(this.parameters, index - 1, thisRow);
Vue.set(this.parameters, index, lastRow)
},
remove: function (index) {
// 移除整行输入控件及内容
this.parameters.splice(index, 1);
this.$emit('change', this.parameters);
},
change: function () {
let isNeedCreate = true;
let removeIndex = -1;
this.parameters.forEach((item, index) => {
if (!item.name && !item.value) {
// 多余的空行
if (index !== this.parameters.length - 1) {
removeIndex = index;
}
// 没有空行，需要创建空行
isNeedCreate = false;
}
});
if (isNeedCreate) {
this.parameters.push(new KeyValue({
type: 'text',
enable: true,
uuid: this.uuid(),
contentType: 'text/plain'
}));
}
this.$emit('change', this.parameters);
// TODO 检查key重复
},
isDisable: function (index) {
return this.parameters.length - 1 == index;
},
querySearch(queryString, cb) {
let suggestions = this.suggestions;
let results = queryString ? suggestions.filter(this.createFilter(queryString)) : suggestions;
cb(results);
},
createFilter(queryString) {
return (restaurant) => {
return (restaurant.value.toLowerCase().indexOf(queryString.toLowerCase()) === 0);
};
},
funcFilter(queryString) {
return (func) => {
return (func.name.toLowerCase().indexOf(queryString.toLowerCase()) > -1);
};
},
uuid: function () {
return (((1 + Math.random()) * 0x100000) | 0).toString(16).substring(1);
},
advanced(item) {
if (item.type === 'json') {
this.$refs.variableJson.open(item);
this.currentItem = item;
} else {
this.$refs.variableAdvance.open();
this.currentItem = item;
}
},
typeChange(item) {
if (item.type === 'file') {
item.contentType = 'application/octet-stream';
} else if (item.type === 'text') {
item.contentType = 'text/plain';
} else {
item.contentType = 'application/json'
}
this.reload();
},
selectAll() {
this.parameters.forEach(item => {
item.enable = true;
});
},
invertSelect() {
this.parameters.forEach(item => {
item.enable = false;
});
},
reload() {
this.isActive = false;
this.$nextTick(() => {
this.isActive = true;
});
},
callback(item) {
this.currentItem.value = item;
this.currentItem = null;
}
},
created() {
if (this.parameters.length === 0 || this.parameters[this.parameters.length - 1].name) {
this.parameters.push(new KeyValue({
type: 'text',
enable: true,
required: true,
uuid: this.uuid(),
contentType: 'text/plain'
}));
}
}
}
</script>
<style scoped>
.kv-description {
font-size: 13px;
}
.kv-row {
margin-top: 10px;
}
.kv-delete {
width: 60px;
}
.kv-select {
width: 50%;
}
.el-autocomplete {
width: 100%;
}
.kv-checkbox {
width: 20px;
margin-right: 10px;
}
.advanced-item-value >>> .el-dialog__body {
padding: 15px 25px;
}
.el-row {
margin-bottom: 5px;
}
.kv-type {
width: 70px;
}
.pointer {
cursor: pointer;
color: #1E90FF;
}
.kv-setting {
width: 40px;
padding: 0px !important;
}
</style>

108
frontend/src/views/system/datasource/CodeEdit.vue Normal file
View File

@ -0,0 +1,108 @@
<template>
<editor v-model="formatData" :lang="mode" @init="editorInit" :theme="theme" :height="height"/>
</template>
<script>
import {formatJson, formatXml} from "./format-utils";
export default {
name: "CodeEdit",
components: { editor: require('vue2-ace-editor')},
data() {
return {
formatData: ''
}
},
props: {
height: [String, Number],
data: {
type: String
},
theme: {
type: String,
default() {
return 'chrome'
}
},
init: {
type: Function
},
enableFormat: {
type: Boolean,
default() {
return true;
}
},
readOnly: {
type: Boolean,
default() {
return false;
}
},
mode: {
type: String,
default() {
return 'text';
}
},
modes: {
type: Array,
default() {
return ['text', 'json', 'xml'];
}
}
},
mounted() {
if (!this.data) {
this.formatData = "";
}
this.format();
},
watch: {
formatData() {
this.$emit('update:data', this.formatData);
},
mode() {
this.format();
}
},
methods: {
editorInit: function (editor) {
require('brace/ext/language_tools') //language extension prerequsite...
this.modes.forEach(mode => {
require('brace/mode/' + mode); //language
});
require('brace/theme/' + this.theme)
require('brace/snippets/javascript') //snippet
if (this.readOnly) {
editor.setReadOnly(true);
}
if (this.init) {
this.init(editor);
}
},
format() {
if (this.enableFormat) {
if (this.data) {
switch (this.mode) {
case 'json':
this.formatData = formatJson(this.data);
break;
case 'xml':
this.formatData = formatXml(this.data);
break;
default:
this.formatData = this.data;
}
}
} else {
this.formatData = this.data;
}
}
}
}
</script>
<style scoped>
</style>

43
frontend/src/views/system/datasource/DialogFooter.vue Normal file
View File

@ -0,0 +1,43 @@
<template>
<div class="dialog-footer">
<el-button @click="cancel" :size="btnSize">{{$t('commons.cancel')}}</el-button>
<el-button type="primary" @click="confirm" @keydown.enter.native.prevent :size="btnSize">{{$t('commons.confirm')}}</el-button>
<el-button type="primary" v-if="isShow" @click="saveAsEdit" @keydown.enter.native.prevent>{{title}}</el-button>
</div>
</template>
<script>
export default {
name: "DialogFooter",
props: {
isShow: {
type: Boolean,
default: false,
},
title:String,
btnSize: {
type: String,
default() {
return '';
}
}
},
methods: {
cancel() {
this.$emit("cancel");
},
confirm() {
this.$emit("confirm");
},
saveAsEdit() {
this.$emit("saveAsEdit");
}
}
}
</script>
<style scoped>
</style>

1
frontend/src/views/system/datasource/DsMain.vue
View File

@ -37,7 +37,6 @@ export default {
methods: {
// 切换main区内容
switchMain(param) {
console.log(param)
const {component, componentParam, tData} = param
this.component = DataHome
this.param = null

21
frontend/src/views/system/datasource/DsTree.vue
View File

@ -39,23 +39,35 @@
>
<span slot-scope="{ node, data }" class="custom-tree-node-list father">
<span style="display: flex;flex: 1;width: 0;">
<span v-if="data.type !== 'folder' && data.status !== 'Error'">
<span v-if="data.type !== 'folder' && data.status !== 'Error' && data.status !== 'Warning'">
<svg-icon icon-class="datasource" class="ds-icon-scene"/>
</span>
<span v-if="data.status === 'Error'">
<svg-icon icon-class="exclamationmark" class="ds-icon-scene"/>
</span>
<span v-if="data.status === 'Warning'">
<svg-icon icon-class="exclamationmark2" class="ds-icon-scene"/>
</span>
<span v-if="data.type === 'folder'">
<i class="el-icon-folder"/>
</span>
<span v-if=" data.status === 'Error'" style="margin-left: 6px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
<span v-if=" data.status === 'Error'"
style="margin-left: 6px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
<el-tooltip effect="dark" :content="$t('datasource.in_valid')" placement="right">
<span>
{{ data.name }}
</span>
</el-tooltip>
</span>
<span v-if=" data.status !== 'Error'"
<span v-if=" data.status === 'Warning'"
style="margin-left: 6px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
<el-tooltip effect="dark" :content="$t('datasource.warning')" placement="right">
<span>
{{ data.name }}
</span>
</el-tooltip>
</span>
<span v-if="data.status !== 'Error' && data.status !== 'Warning'"
style="margin-left: 6px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
{{ data.name }}
</span>
@ -212,6 +224,8 @@ export default {
return 'Apache Hive'
} else if (type === 'db2') {
return 'Db2'
} else if (type === 'api') {
return 'API'
}
},
@ -268,7 +282,6 @@ export default {
})
},
switchMain(component, componentParam, tData) {
console.log(tData)
this.$emit('switch-main', {
component,
componentParam,

86
frontend/src/views/system/datasource/EditDialog.vue Normal file
View File

@ -0,0 +1,86 @@
<template>
<el-dialog :close-on-click-modal="closeOnClickModal"
:title="title"
:width="width"
:visible="visible"
destroy-on-close
:append-to-body="appendToBody"
@close="handleClose">
<slot name="header"></slot>
<slot></slot>
<template v-slot:footer>
<slot name="footer">
<div v-if="withFooter" class="dialog-footer">
<ms-dialog-footer
@cancel="handleCancel"
@confirm="handleConfirm"/>
</div>
</slot>
</template>
</el-dialog>
</template>
<script>
import DialogFooter from "./DialogFooter";
export default {
name: "EditDialog",
components: {DialogFooter},
props: {
title: {
type: String,
default() {
return 'title';
}
},
visible: {
type: Boolean,
default() {
return false;
}
},
appendToBody: {
type: Boolean,
default() {
return false;
}
},
width: {
type: String,
default() {
return "50%";
}
},
withFooter: {
type: Boolean,
default() {
return true;
}
},
closeOnClickModal: {
type: Boolean,
default: false
},
},
methods: {
handleConfirm() {
this.$emit('confirm');
},
handleCancel() {
this.handleClose();
this.$emit('cancel');
},
handleClose() {
this.$emit('update:visible', false);
this.$emit('close');
},
}
};
</script>
<style scoped>
</style>

118
frontend/src/views/system/datasource/ImportJson.vue Normal file
View File

@ -0,0 +1,118 @@
<template>
<el-dialog
:title="$t('commons.import')"
:visible.sync="importVisible"
width="50%"
append-to-body
show-close
:close-on-click-modal="false"
@closed="handleClose">
<el-tabs v-model="activeName">
<el-tab-pane label="JSON" name="JSON">
<div style="height: 400px">
<ms-code-edit :mode="mode"
:data.sync="json" theme="eclipse" :modes="[]"
ref="codeEdit"/>
</div>
</el-tab-pane>
<el-tab-pane label="JSON-SCHEMA" name="JSON-SCHEMA">
<div style="height: 400px">
<ms-code-edit :mode="mode"
:data.sync="jsonSchema" theme="eclipse" :modes="[]"
ref="codeEdit"/>
</div>
</el-tab-pane>
</el-tabs>
<span slot="footer" class="dialog-footer">
<ms-dialog-footer
@cancel="importVisible = false"
@confirm="saveConfirm"/>
</span>
</el-dialog>
</template>
<script>
import DialogFooter from './DialogFooter'
import CodeEdit from "./CodeEdit";
import json5 from 'json5';
const Convert = require('./convert.js');
const MsConvert = new Convert();
export default {
name: "MsImportJson",
components: {DialogFooter, CodeEdit},
data() {
return {
importVisible: false,
activeName: "JSON",
mode: "json",
json: "",
jsonSchema: "",
};
},
watch: {},
props: {},
methods: {
openOneClickOperation() {
this.importVisible = true;
},
checkIsJson(json) {
try {
json5.parse(json);
return true;
} catch (e) {
return false;
}
},
checkIsJsonSchema(json) {
try {
json = json5.parse(json);
if (json.properties && typeof json.properties === 'object' && !json.type) {
json.type = 'object';
}
if (json.items && typeof json.items === 'object' && !json.type) {
json.type = 'array';
}
if (!json.type) {
return false;
}
json.type = json.type.toLowerCase();
let types = ['object', 'string', 'number', 'array', 'boolean', 'integer'];
if (types.indexOf(json.type) === -1) {
return false;
}
return JSON.stringify(json);
} catch (e) {
return false;
}
},
saveConfirm() {
if (this.activeName === 'JSON') {
if (!this.checkIsJson(this.json)) {
this.$error(this.$t('schema.json_warning'));
return;
}
let jsonData = MsConvert.format(json5.parse(this.json));
//let jsonData = GenerateSchema(json5.parse(this.json));
this.$emit('jsonData', jsonData);
} else {
if (!this.checkIsJsonSchema(this.jsonSchema)) {
this.$error(this.$t('schema.json_schema_warning'));
return;
}
let obj = json5.parse(this.jsonSchema);
this.$emit('jsonData', obj);
}
this.importVisible = false;
},
handleClose() {
this.importVisible = false;
},
}
}
</script>
<style scoped>
</style>

206
frontend/src/views/system/datasource/convert.js Normal file
View File

@ -0,0 +1,206 @@
const isBoolean = require("lodash.isboolean");
const isEmpty = require("lodash.isempty");
const isInteger = require("lodash.isinteger");
const isNull = require("lodash.isnull");
const isNumber = require("lodash.isnumber");
const isObject = require("lodash.isobject");
const isString = require("lodash.isstring");
const isArray = Array.isArray;
class Convert {
constructor() {
this._option = {
$id: "http://example.com/root.json",
$schema: "http://json-schema.org/draft-07/schema#",
}
this._object = null;
}
/**
* 转换函数
* @param {*} object 需要转换的对象
* @param {*} ?option 可选参数，目前只有能设置 root 节点的 $id 和 $schema
*/
format(object, option = {}) {
// 数据校验，确保传入的的object只能是对象或数组
if (!isObject(object)) {
throw new TypeError("传入参数只能是对象或数组");
}
// 合并属性
this._option = Object.assign(this._option, option);
// 需要转换的对象
this._object = object;
let convertRes;
// 数组类型和对象类型结构不一样
if (isArray(object)) {
convertRes = this._arrayToSchema();
} else {
convertRes = this._objectToSchema();
}
// 释放
this._object = null;
return convertRes;
}
/**
* 数组类型转换成JSONSCHEMA
*/
_arrayToSchema() {
// root节点基本信息
let result = this._value2object(this._object, this._option.$id, "", true);
if (this._object.length > 0) {
let itemArr = [];
for (let index = 0; index < this._object.length; index++) {
// 创建items对象的基本信息
let objectItem = this._object[index]
let item = this._value2object(objectItem, `#/items`, 'items');
if (isObject(objectItem) && !isEmpty(objectItem)) {
// 递归遍历
let objectItemSchema = this._json2schema(objectItem, `#/items`);
// 合并对象
item = Object.assign(item, objectItemSchema);
}
itemArr.push(item);
}
result["items"] = itemArr;
}
return result
}
/**
* 对象类型转换成JSONSCHEMA
*/
_objectToSchema() {
let baseResult = this._value2object(this._object, this._option.$id, "", true)
let objectSchema = this._json2schema(this._object)
baseResult = Object.assign(baseResult, objectSchema)
return baseResult
}
/**
* 递归函数，转换object对象为json schmea 格式
* @param {*} object 需要转换对象
* @param {*} name $id值
*/
_json2schema(object, name = "") {
// 如果递归值不是对象，那么return掉
if (!isObject(object)) {
return;
}
// 处理当前路径$id
if (name === "" || name == undefined) {
name = "#"
}
let result = {};
// 判断传入object是对象还是数组。
if (isArray(object)) {
result.items = {};
} else {
result.properties = {};
}
// 遍历传入的对象
for (const key in object) {
if (object.hasOwnProperty(key)) {
const element = object[key];
// 如果只是undefined。跳过
if (element === undefined) {
continue;
}
let $id = `${name}/properties/${key}`
// 判断当前 element 的值 是否也是对象，如果是就继续递归，不是就赋值给result
if(!result["properties"]){
continue;
}
if (isObject(element)) {
// 创建当前属性的基本信息
result["properties"][key] = this._value2object(element, $id, key)
if (isArray(element)) {
// 针对空数组和有值的数组做不同处理
if (element.length > 0) {
// 是数组
let itemArr = [];
for (let index = 0; index < element.length; index++) {
let elementItem = element[index];
// 创建items对象的基本信息
let item = this._value2object(elementItem, `${$id}/items`, key + 'items');
// 判断第一项是否是对象,且对象属性不为空
if (isObject(elementItem) && !isEmpty(elementItem)) {
// 新增的properties才合并进来
item = Object.assign(item, this._json2schema(elementItem, `${$id}/items`));
}
itemArr.push(item);
}
result["properties"][key]["items"] = itemArr;
}
} else {
// 不是数组，递归遍历获取，然后合并对象属性
result["properties"][key] = Object.assign(result["properties"][key], this._json2schema(element, $id));
}
} else {
// 一般属性直接获取基本信息
if (result["properties"]) {
result["properties"][key] = this._value2object(element, $id, key);
}
}
}
}
return result;
}
/**
* 把json的值转换成对象类型
* @param {*} value
* @param {*} $id
* @param {*} key
*/
_value2object(value, $id, key = '', root = false) {
let objectTemplate = {
$id: $id,
title: `The ${key} Schema`,
mock: {
"mock": value
},
}
// 判断是否为初始化root数据
if (root) {
objectTemplate["$schema"] = this._option.$schema;
objectTemplate["title"] = `The Root Schema`;
objectTemplate["mock"] = undefined;
}
if (isBoolean(value)) {
objectTemplate.type = "boolean";
} else if (isInteger(value)) {
objectTemplate.type = "integer";
} else if (isNumber(value)) {
objectTemplate.type = "number";
} else if (isString(value)) {
objectTemplate.type = "string";
} else if (isNull(value)) {
objectTemplate.type = "null";
} else if (isArray(value)) {
objectTemplate.type = "array";
objectTemplate["mock"] = undefined;
} else if (isObject(value)) {
objectTemplate.type = "object"
objectTemplate["mock"] = undefined;
}
return objectTemplate;
}
/**
* 后台转换
* @param callback
*/
// schemaToJsonStr(schema, callback) {
// post('/api/definition/preview', schema, (response) => {
// if (callback) {
// callback(response.data);
// }
// });
// }
}
module.exports = Convert;

468
frontend/src/views/system/datasource/form.vue
View File

@ -30,6 +30,7 @@
class="select-width"
:disabled="formType=='modify' || (formType==='add' && params && !!params.type)"
@change="changeType()"
filterable
>
<el-option
v-for="item in allTypes"
@ -40,104 +41,210 @@
</el-select>
</el-form-item>
<el-form-item
v-if="form.configuration.dataSourceType=='jdbc'"
:label="$t('datasource.host')"
prop="configuration.host"
>
<el-form-item v-if="form.type == 'api'" :label="$t('datasource.data_table')">
<el-col>
<el-button size="mini" icon="el-icon-plus" type="text" @click="addApiItem(undefined)"/>
<el-table :data="form.apiConfiguration" class="my_table" max-height="300" height="300">
<el-table-column prop="name" :label="$t('datasource.data_table_name')" width="150" show-overflow-tooltip></el-table-column>
<el-table-column prop="method" :label="$t('datasource.method')" width="150" show-overflow-tooltip></el-table-column>
<el-table-column prop="url" :label="$t('datasource.url')" width="150" show-overflow-tooltip></el-table-column>
<el-table-column :label="$t('dataset.operate')">
<template slot-scope="scope" style="float: right">
<el-button size="mini" type="primary" icon="el-icon-edit" circle @click="addApiItem(scope.row)"/>
<el-button size="mini" type="danger" icon="el-icon-delete" circle @click="deleteItem(scope.row)"/>
</template>
</el-table-column>
</el-table>
</el-col>
</el-form-item>
<el-dialog v-dialogDrag :title="api_table_title" :visible="edit_api_item" :before-close="closeEditItem" width="70%" class="dialog-css" append-to-body>
<el-steps :active="active" align-center>
<el-step title="步骤 1"></el-step>
<el-step title="步骤 2"></el-step>
</el-steps>
<el-row v-show="active === 1">
<el-form ref="apiItem" :model="apiItem" label-width="100px" :rules="rule">
<p class="tip">{{ $t('datasource.base_info') }} </p>
<el-form-item :label="$t('commons.name')" prop="name">
<el-input v-model="apiItem.name" autocomplete="off"/>
</el-form-item>
<el-form-item :label="$t('datasource.request')" prop="url">
<el-input :placeholder="$t('datasource.path_all_info')" v-model="apiItem.url" class="ms-http-input" size="small" >
<el-select v-model="apiItem.method" slot="prepend" style="width: 100px" size="small">
<el-option v-for="item in reqOptions" :key="item.id" :label="item.label" :value="item.id"/>
</el-select>
</el-input>
</el-form-item>
<div v-loading="loading">
<p class="tip">{{ $t('datasource.req_param') }} </p>
<!-- HTTP 请求参数 -->
<el-form-item>
<api-http-request-form :headers="apiItem.request.headers" :request="apiItem.request" :response="responseData"/>
</el-form-item>
</div>
<el-form-item :label="$t('datasource.data_path')" pprop="dataPath">
<el-input :placeholder="$t('datasource.data_path_desc')" v-model="apiItem.dataPath" autocomplete="off"/>
</el-form-item>
<el-button style="margin-top: 12px;" @click="validateApi(undefined)" v-show="active === 1">{{ $t('commons.validate') }}</el-button>
</el-form>
</el-row>
<el-row v-show="active === 2">
<el-tabs v-model="api_step2_active_name" @tab-click="handleClick">
<el-tab-pane label="数据预览" name="first">
<ux-grid ref="plxTable" size="mini" style="width: 100%;" :height="height" :checkbox-config="{highlight: true}" :width-resize="true" >
<ux-table-column v-for="field in apiItem.fields" :key="field.originName" min-width="200px" :field="field.originName" :resizable="true">
<template slot="header">
<svg-icon v-if="field.deExtractType === 0" icon-class="field_text" class="field-icon-text" />
<svg-icon v-if="field.deExtractType === 1" icon-class="field_time" class="field-icon-time" />
<svg-icon v-if="field.deExtractType === 2 || field.deExtractType === 3" icon-class="field_value" class="field-icon-value" />
<svg-icon v-if="field.deExtractType === 5" icon-class="field_location" class="field-icon-location" />
<span>{{ field.name }}</span>
</template>
</ux-table-column>
</ux-grid>
</el-tab-pane>
<el-tab-pane label="字段管理" name="second">
<el-table :data="apiItem.fields" size="mini">
<el-table-column property="name" :label="$t('dataset.field_name')" width="180">
<template slot-scope="scope">
<el-input v-model="scope.row.name" size="mini"/>
</template>
</el-table-column>
<el-table-column property="originName" :label="$t('dataset.field_origin_name')" width="100">
<template slot-scope="scope">
<span v-if="scope.row.extField === 0" :title="scope.row.originName" class="field-class" style="width: 100%;white-space: nowrap;overflow: hidden;text-overflow: ellipsis;">
<span style="font-size: 12px;">{{ scope.row.originName }}</span>
</span>
</template>
</el-table-column>
<el-table-column property="deExtractType" :label="$t('dataset.field_type')" width="140">
<template slot-scope="scope">
<el-select v-model="scope.row.deExtractType" size="mini" style="display: inline-block;width: 26px;">
<el-option v-for="item in fieldTypes" :key="item.value" :label="item.label" :value="item.value">
<span style="float: left">
<svg-icon v-if="item.value === 0" icon-class="field_text" class="field-icon-text" />
<svg-icon v-if="item.value === 1" icon-class="field_time" class="field-icon-time" />
<svg-icon v-if="item.value === 2 || item.value === 3" icon-class="field_value" class="field-icon-value" />
<svg-icon v-if="item.value === 5" icon-class="field_location" class="field-icon-location" />
</span>
<span style="float: left; color: #8492a6; font-size: 12px">{{ item.label }}</span>
</el-option>
</el-select>
<span style="margin-left: 8px;">
<span v-if="scope.row.deExtractType === 0">
<svg-icon icon-class="field_text" class="field-icon-text" />
<span class="field-class">{{ $t('dataset.text') }}</span>
</span>
<span v-if="scope.row.deExtractType === 1">
<svg-icon v-if="scope.row.deExtractType === 1" icon-class="field_time" class="field-icon-time" />
<span class="field-class">{{ $t('dataset.time') }}</span>
</span>
<span v-if="scope.row.deExtractType === 2 || scope.row.deExtractType === 3">
<svg-icon v-if="scope.row.deExtractType === 2 || scope.row.deExtractType === 3" icon-class="field_value" class="field-icon-value" />
<span v-if="scope.row.deExtractType === 2" class="field-class">{{ $t('dataset.value') }}</span>
<span v-if="scope.row.deExtractType === 3" class="field-class">{{ $t('dataset.value') + '(' + $t('dataset.float') + ')' }}</span>
</span>
<span v-if="scope.row.deExtractType === 5">
<svg-icon v-if="scope.row.deExtractType === 5" icon-class="field_location" class="field-icon-location" />
<span class="field-class">{{ $t('dataset.location') }}</span>
</span>
</span>
</template>
</el-table-column>
</el-table>
</el-tab-pane>
</el-tabs>
</el-row>
<div slot="footer" class="dialog-footer">
<el-button style="margin-top: 12px;" @click="next" v-show="active === 1">{{ $t('fu.steps.next') }}</el-button>
<el-button style="margin-top: 12px;" @click="before" v-show="active === 2">{{ $t('fu.steps.prev') }}</el-button>
<el-button style="margin-top: 12px;" @click="saveItem" v-show="active === 2">{{ $t('commons.save') }}</el-button>
</div>
</el-dialog>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'"
:label="$t('datasource.host')" prop="configuration.host">
<el-input v-model="form.configuration.host" autocomplete="off"/>
</el-form-item>
<el-form-item
v-if="form.configuration.dataSourceType=='es'"
:label="$t('datasource.datasource_url')"
prop="configuration.url"
>
<el-input
v-model="form.configuration.url"
:placeholder="$t('datasource.please_input_datasource_url')"
autocomplete="off"
/>
<el-form-item v-if="form.configuration.dataSourceType=='es' && form.type !== 'api'"
:label="$t('datasource.datasource_url')" prop="configuration.url">
<el-input v-model="form.configuration.url" :placeholder="$t('datasource.please_input_datasource_url')"
autocomplete="off"/>
</el-form-item>
<el-form-item
v-if="form.configuration.dataSourceType=='jdbc'"
:label="$t('datasource.data_base')"
prop="configuration.dataBase"
>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'"
:label="$t('datasource.data_base')" prop="configuration.dataBase">
<el-input v-model="form.configuration.dataBase" autocomplete="off"/>
</el-form-item>
<el-form-item
v-if="form.type=='oracle'"
:label="$t('datasource.oracle_connection_type')"
prop="configuration.connectionType"
>
<el-form-item v-if="form.type=='oracle' && form.type !== 'api'" :label="$t('datasource.oracle_connection_type')"
prop="configuration.connectionType">
<el-radio v-model="form.configuration.connectionType" label="sid">{{ $t('datasource.oracle_sid') }}</el-radio>
<el-radio v-model="form.configuration.connectionType" label="serviceName">
{{ $t('datasource.oracle_service_name') }}
</el-radio>
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.user_name')">
<el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'"
:label="$t('datasource.user_name')">
<el-input v-model="form.configuration.username" autocomplete="off"/>
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.password')">
<el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'"
:label="$t('datasource.password')">
<el-input v-model="form.configuration.password" autocomplete="off" show-password/>
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='es'" :label="$t('datasource.user_name')">
<el-form-item v-if="form.configuration.dataSourceType=='es' && form.type !== 'api'"
:label="$t('datasource.user_name')">
<el-input v-model="form.configuration.esUsername" autocomplete="off"/>
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='es'" :label="$t('datasource.password')">
<el-form-item v-if="form.configuration.dataSourceType=='es' && form.type !== 'api'"
:label="$t('datasource.password')">
<el-input v-model="form.configuration.esPassword" autocomplete="off" show-password/>
</el-form-item>
<el-form-item
v-if="form.configuration.dataSourceType=='jdbc' && form.type!=='oracle'"
:label="$t('datasource.extra_params')"
>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type!=='oracle' && form.type !== 'api'"
:label="$t('datasource.extra_params')">
<el-input v-model="form.configuration.extraParams" autocomplete="off"/>
</el-form-item>
<el-form-item
v-if="form.configuration.dataSourceType=='jdbc'"
:label="$t('datasource.port')"
prop="configuration.port"
>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'"
:label="$t('datasource.port')" prop="configuration.port">
<el-input v-model="form.configuration.port" autocomplete="off" type="number" min="0"/>
</el-form-item>
<el-form-item v-if="form.type=='oracle' || form.type=='sqlServer' || form.type=='pg' || form.type=='redshift' || form.type=='db2'">
<el-button icon="el-icon-plus" size="mini" @click="getSchema()">
{{ $t('datasource.get_schema') }}
</el-button>
<el-form-item
v-if="form.type=='oracle' || form.type=='sqlServer' || form.type=='pg' || form.type=='redshift' || form.type=='db2'">
<el-button icon="el-icon-plus" size="mini" @click="getSchema()">{{ $t('datasource.get_schema') }}</el-button>
</el-form-item>
<el-form-item
v-if="form.type=='oracle' || form.type=='sqlServer' || form.type=='pg' || form.type=='redshift' || form.type=='db2'"
:label="$t('datasource.schema')"
>
<el-select
v-model="form.configuration.schema"
filterable
:placeholder="$t('datasource.please_choose_schema')"
class="select-width"
>
<el-option
v-for="item in schemas"
:key="item"
:label="item"
:value="item"
/>
:label="$t('datasource.schema')">
<el-select v-model="form.configuration.schema" filterable :placeholder="$t('datasource.please_choose_schema')"
class="select-width">
<el-option v-for="item in schemas" :key="item" :label="item" :value="item"/>
</el-select>
</el-form-item>
<el-collapse v-if="form.configuration.dataSourceType=='jdbc'">
<el-collapse v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'">
<el-collapse-item :title="$t('datasource.priority')" name="1">
<el-form-item :label="$t('datasource.initial_pool_size')" prop="configuration.initialPoolSize">
<el-input
v-model="form.configuration.initialPoolSize"
autocomplete="off"
type="number"
min="0"
size="small"
/>
<el-input v-model="form.configuration.initialPoolSize" autocomplete="off" type="number" min="0"
size="small"/>
</el-form-item>
<el-form-item :label="$t('datasource.min_pool_size')" prop="configuration.minPoolSize">
<el-input v-model="form.configuration.minPoolSize" autocomplete="off" type="number" min="0"/>
@ -145,34 +252,23 @@
<el-form-item :label="$t('datasource.max_pool_size')" prop="configuration.maxPoolSize">
<el-input v-model="form.configuration.maxPoolSize" autocomplete="off" type="number" min="0"/>
</el-form-item>
</el-collapse-item>
</el-collapse>
</el-form>
<div v-if="canEdit" slot="footer" class="dialog-footer">
<el-button
v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
@click="validaDatasource"
>{{ $t('commons.validate') }}
<el-button v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
@click="validaDatasource">{{ $t('commons.validate') }}
</el-button>
<el-button
v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
type="primary"
@click="save"
>{{ $t('commons.save') }}
<el-button v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)" type="primary"
@click="save">{{ $t('commons.save') }}
</el-button>
</div>
<div v-else slot="footer" class="dialog-footer">
<el-button
v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
@click="validaDatasource"
>{{ $t('commons.validate') }}
<el-button v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
@click="validaDatasource">{{ $t('commons.validate') }}
</el-button>
<el-button
v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
type="primary"
@click="changeEdit"
>{{ $t('commons.edit') }}
<el-button v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)" type="primary"
@click="changeEdit">{{ $t('commons.edit') }}
</el-button>
</div>
</div>
@ -181,13 +277,17 @@
<script>
import LayoutContent from '@/components/business/LayoutContent'
import {addDs, editDs, getSchema, validateDs, validateDsById} from '@/api/system/datasource'
import {addDs, editDs, getSchema, validateDs, validateDsById, checkApiDatasource} from '@/api/system/datasource'
import {$confirm} from '@/utils/message'
import i18n from '@/lang/index'
import ApiHttpRequestForm from '@/views/system/datasource/ApiHttpRequestForm'
export default {
name: 'DsForm',
components: {LayoutContent},
components: {
LayoutContent,
ApiHttpRequestForm
},
props: {
params: {
type: Object,
@ -210,7 +310,8 @@ export default {
acquireIncrement: 5,
idleConnectionTestPeriod: 5,
connectTimeout: 5
}
},
apiConfiguration: []
},
rule: {
name: [{required: true, message: i18n.t('datasource.input_name'), trigger: 'blur'},
@ -269,26 +370,81 @@ export default {
required: true,
message: i18n.t('datasource.please_input_connect_timeout'),
trigger: 'change'
}]
}],
'url': [{required: true, message: i18n.t('datasource.please_input_url'), trigger: 'change'}],
'dataPath': [{required: true, message: i18n.t('datasource.please_input_dataPath'), trigger: 'change'}]
},
allTypes: [
{name: 'mysql', label: 'MySQL', type: 'jdbc', extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
{
name: 'mysql',
label: 'MySQL',
type: 'jdbc',
extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
},
{name: 'hive', label: 'Apache Hive', type: 'jdbc', extraParams: ''},
{name: 'oracle', label: 'Oracle', type: 'jdbc'},
{name: 'sqlServer', label: 'SQL Server', type: 'jdbc', extraParams: ''},
{name: 'pg', label: 'PostgreSQL', type: 'jdbc', extraParams: ''},
{name: 'es', label: 'Elasticsearch', type: 'es'},
{name: 'mariadb', label: 'MariaDB', type: 'jdbc', extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'},
{name: 'ds_doris', label: 'Doris', type: 'jdbc', extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'},
{
name: 'mariadb',
label: 'MariaDB',
type: 'jdbc',
extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
},
{
name: 'ds_doris',
label: 'Doris',
type: 'jdbc',
extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
},
{name: 'ck', label: 'ClickHouse', type: 'jdbc', extraParams: ''},
{name: 'redshift', label: 'AWS Redshift', type: 'jdbc'},
{name: 'mongo', label: 'MongoDB', type: 'jdbc', extraParams: ''},
{name: 'db2', label: 'Db2', type: 'jdbc', extraParams: ''}
{name: 'db2', label: 'Db2', type: 'jdbc', extraParams: ''},
{name: 'api', label: 'API', type: 'api', extraParams: ''}
],
schemas: [],
canEdit: false,
originConfiguration: {}
originConfiguration: {},
edit_api_item: false,
add_api_item: false,
active: 0,
defaultApiItem: {
name: '',
url: '',
method: 'GET',
request: {
headers: [],
body: {}
},
fields: []
},
apiItem: {
name: '',
url: '',
method: 'GET',
dataPath: '',
request: {
headers: [],
body: {},
authManager: {}
},
fields: []
},
reqOptions: [{id: 'GET', label: 'GET'}, {id: 'POST', label: 'POST'}],
loading: false,
responseData: {type: 'HTTP', responseResult: {}, subRequestResults: []},
api_table_title: '',
api_step2_active_name: 'first',
fieldTypes: [
{ label: this.$t('dataset.text'), value: 0 },
{ label: this.$t('dataset.time'), value: 1 },
{ label: this.$t('dataset.value'), value: 2 },
{ label: this.$t('dataset.value') + '(' + this.$t('dataset.float') + ')', value: 3 },
{ label: this.$t('dataset.location'), value: 5 }
],
height: 500
}
},
@ -330,9 +486,13 @@ export default {
},
edit(row) {
this.formType = 'modify'
this.form = Object.assign({}, row)
this.form = JSON.parse(JSON.stringify(row))
this.originConfiguration = this.form.configuration
if(row.type === 'api'){
}else {
this.form.configuration = JSON.parse(this.form.configuration)
}
},
reset() {
this.$refs.dsForm.resetFields()
@ -401,7 +561,12 @@ export default {
}
const method = this.formType === 'add' ? addDs : editDs
const form = JSON.parse(JSON.stringify(this.form))
if(form.type === 'api'){
form.configuration = JSON.stringify(form.apiConfiguration)
}else {
form.configuration = JSON.stringify(form.configuration)
}
if (this.formType === 'modify' && this.originConfiguration !== form.configuration) {
if (repeat) {
$confirm(i18n.t('datasource.repeat_datasource_msg') + '[' + repeatDsName.join(',') + '], ' + i18n.t('datasource.confirm_save'), () => {
@ -505,6 +670,104 @@ export default {
},
refreshType(form) {
this.$emit('refresh-type', form)
},
next() {
if(this.active === 1){
this.$refs.apiItem.validate(valid => {
if (valid) {
console.log(this.apiItem)
const data = JSON.parse(JSON.stringify(this.apiItem))
data.request = JSON.stringify(data.request)
this.loading = true
checkApiDatasource(data).then(res => {
this.loading = false
console.log(res)
this.$success(i18n.t('commons.success'))
this.active++
this.apiItem.fields = res.data.fields
this.$refs.plxTable.reloadData(res.data.datas)
}).catch(res => {
this.apiItem.fields = []
this.loading = false
})
} else {
this.apiItem.fields = []
return false
}
})
}
},
before() {
this.active--
},
closeEditItem() {
this.active = 0
this.edit_api_item = false
},
saveItem() {
this.active = 0
this.edit_api_item = false
if(!this.add_api_item){
this.form.apiConfiguration.push(this.apiItem)
}
},
addApiItem(item) {
if (item) {
this.add_api_item = true
this.api_table_title = this.$t('datasource.edit_api_table')
this.apiItem = item
}else {
this.add_api_item = false
this.apiItem = JSON.parse(JSON.stringify(this.defaultApiItem))
this.api_table_title = this.$t('datasource.add_api_table')
}
this.active = 1
this.edit_api_item = true
},
deleteItem(item) {
this.form.apiConfiguration.splice(this.form.apiConfiguration.indexOf(item), 1)
},
runDebug() {
this.$refs['debugForm'].validate((valid) => {
if (valid) {
this.loading = true;
this.isStop = true;
this.request.url = this.debugForm.url;
this.request.method = this.debugForm.method;
this.request.name = getUUID().substring(0, 8);
this.runData = [];
this.runData.push(this.request);
/*触发执行操作*/
this.reportId = getUUID().substring(0, 8);
}
})
},
validateApi(item) {
if(undefined){
}else {
this.$refs.apiItem.validate(valid => {
if (valid) {
const data = JSON.parse(JSON.stringify(this.apiItem))
data.request = JSON.stringify(data.request)
this.loading = true
checkApiDatasource(data).then(res => {
this.loading = false
console.log(res)
this.$success(i18n.t('commons.success'))
this.apiItem.fields = res.data.fields
this.$refs.plxTable.reloadData(res.data.datas)
}).catch(res => {
this.loading = false
})
} else {
return false
}
})
}
},
handleClick(tab, event) {
console.log(tab, event);
}
}
}
@ -520,12 +783,27 @@ export default {
transform: scale(0.85);
}
}
.el-input {
width: 300px;
}
.el-select {
width: 300px;
}
.ms-http-input {
width: 500px;
margin-top: 5px;
}
.tip {
padding: 3px 5px;
font-size: 16px;
border-radius: 0;
border-left: 4px solid #783887;
margin: 5px 5px 10px 5px;
}
.el-select>>>input{
padding-right: 10px;
}
.el-select>>>.el-input__suffix{
right: 0;
}
</style>

185
frontend/src/views/system/datasource/format-utils.js Normal file
View File

@ -0,0 +1,185 @@
export function formatJson (json) {
let i = 0,
il = 0,
tab = " ",
newJson = "",
indentLevel = 0,
inString = false,
currentChar = null;
let flag = false;
for (i = 0, il = json.length; i < il; i += 1) {
currentChar = json.charAt(i);
switch (currentChar) {
case '{':
if (i != 0 && json.charAt(i - 1) === '$') {
newJson += currentChar;
flag = true;
} else if (!inString) {
newJson += currentChar + "\n" + repeat(tab, indentLevel + 1);
indentLevel += 1
} else {
newJson += currentChar
}
break;
case '[':
if (!inString) {
newJson += currentChar + "\n" + repeat(tab, indentLevel + 1);
indentLevel += 1
} else {
newJson += currentChar
}
break;
case '}':
if (flag) {
newJson += currentChar;
flag = false;
} else if (!inString) {
indentLevel -= 1;
newJson += "\n" + repeat(tab, indentLevel) + currentChar
} else {
newJson += currentChar
}
break;
case ']':
if (!inString) {
indentLevel -= 1;
newJson += "\n" + repeat(tab, indentLevel) + currentChar
} else {
newJson += currentChar
}
break;
case ',':
if (!inString) {
newJson += ",\n" + repeat(tab, indentLevel)
} else {
newJson += currentChar
}
break;
case ':':
if (!inString) {
newJson += ": "
} else {
newJson += currentChar
}
break;
case ' ':
case "\n":
case "\t":
if (inString) {
newJson += currentChar
}
break;
case '"':
if (i > 0 && json.charAt(i - 1) !== '\\') {
inString = !inString
}
newJson += currentChar;
break;
default:
newJson += currentChar;
break
}
}
return newJson;
}
function repeat(s, count) {
return new Array(count + 1).join(s)
}
export function formatXml(text) {
//去掉多余的空格
text = '\n' + text.replace(/(<\w+)(\s.*?>)/g, function ($0, name, props) {
return name + ' ' + props.replace(/\s+(\w+=)/g, " $1");
});
//把注释编码
text = text.replace(/<!--(.+?)-->/g, function ($0, text) {
var ret = '<!--' + escape(text) + '-->';
//alert(ret);
return ret;
});
//调整格式
var rgx = /\n(<(([^\?]).+?)(?:\s|\s*?>|\s*?(\/)>)(?:.*?(?:(?:(\/)>)|(?:<(\/)\2>)))?)/mg;
var nodeStack = [];
var output = text.replace(rgx, function ($0, all, name, isBegin, isCloseFull1, isCloseFull2, isFull1, isFull2) {
var isClosed = (isCloseFull1 == '/') || (isCloseFull2 == '/' ) || (isFull1 == '/') || (isFull2 == '/');
//alert([all,isClosed].join('='));
var prefix = '';
if (isBegin == '!') {
prefix = getPrefix(nodeStack.length);
}
else {
if (isBegin != '/') {
prefix = getPrefix(nodeStack.length);
if (!isClosed) {
nodeStack.push(name);
}
}
else {
nodeStack.pop();
prefix = getPrefix(nodeStack.length);
}
}
var ret = '\n' + prefix + all;
return ret;
});
var prefixSpace = -1;
var outputText = output.substring(1);
//把注释还原并解码，调格式
outputText = outputText.replace(/(\s*)<!--(.+?)-->/g, function ($0, prefix, text) {
if (prefix.charAt(0) == '\r')
prefix = prefix.substring(1);
text = unescape(text).replace(/\r/g, '\n');
var ret = '\n' + prefix + '<!--' + text.replace(/^\s*/mg, prefix) + '-->';
//alert(ret);
return ret;
});
return outputText.replace(/\s+$/g, '').replace(/\r/g, '\r\n');
}
/**
* @param time 时间
* @param cFormat 格式
* @returns {string|null} 字符串
* @example formatTime('2018-1-29', '{y}/{m}/{d} {h}:{i}:{s}') // -> 2018/01/29 00:00:00
*/
export function formatTime(time, cFormat) {
if (arguments.length === 0) return null;
if ((time + '').length === 10) {
time = +time * 1000;
}
let format = cFormat || '{y}-{m}-{d} {h}:{i}:{s}', date;
if (typeof time === 'object') {
date = time;
} else {
date = new Date(time);
}
let formatObj = {
y: date.getFullYear(),
m: date.getMonth() + 1,
d: date.getDate(),
h: date.getHours(),
i: date.getMinutes(),
s: date.getSeconds(),
a: date.getDay()
};
return format.replace(/{([ymdhisa])+}/g, (result, key) => {
let value = formatObj[key];
if (key === 'a') return ['一', '二', '三', '四', '五', '六', '日'][value - 1];
if (result.length > 0 && value < 10) {
value = '0' + value;
}
return value || 0;
});
}
function getPrefix(prefixIndex) {
var span = ' ';
var output = [];
for (var i = 0; i < prefixIndex; ++i) {
output.push(span);
}
return output.join('');
}