center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge pull request #6000 from AndroidBaby1/dev

Browse Source 
【数据源】金仓数据源插件关于数据集创建失败问题修复
This commit is contained in:
fit2cloudrd 2023-08-30 16:10:19 +08:00 committed by GitHub
commit a0307fc869
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 420 additions and 616 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -1756,6 +1756,7 @@ public class DataSetTableService {
if (StringUtils.isNotEmpty(schema) && (StringUtils.equalsIgnoreCase(ds.getType(), DatasourceTypes.db2.getType()) ||
StringUtils.equalsIgnoreCase(ds.getType(), DatasourceTypes.sqlServer.getType()) ||
StringUtils.equalsIgnoreCase(ds.getType(), DatasourceTypes.oracle.getType()) ||
StringUtils.equalsIgnoreCase(ds.getType(), DatasourceTypes.kingbase.getType()) ||
StringUtils.equalsIgnoreCase(ds.getType(), DatasourceTypes.pg.getType()))) {
joinPrefix = String.format(keyword, schema) + ".";
}

84
extensions/dataease-extensions-datasource/kingbase/kingbase-backend/src/main/java/io/dataease/plugins/datasource/kingbase/provider/KingbaseDsProvider.java
View File

@ -3,7 +3,6 @@ package io.dataease.plugins.datasource.kingbase.provider;
import com.google.gson.Gson;
import io.dataease.plugins.common.base.domain.DeDriver;
import io.dataease.plugins.common.base.mapper.DeDriverMapper;
import io.dataease.plugins.common.constants.DatasourceTypes;
import io.dataease.plugins.common.dto.datasource.TableDesc;
import io.dataease.plugins.common.dto.datasource.TableField;
import io.dataease.plugins.common.exception.DataEaseException;
@ -15,7 +14,6 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.lang.reflect.Method;
import java.sql.*;
import java.util.ArrayList;
import java.util.LinkedList;
@ -46,43 +44,20 @@ public class KingbaseDsProvider extends DefaultJdbcProvider {
KingbaseConfig kingbaseConfig = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(),
KingbaseConfig.class);
String username = kingbaseConfig.getUsername();
String password = kingbaseConfig.getPassword();
String defaultDriver = kingbaseConfig.getDriver();
String customDriver = kingbaseConfig.getCustomDriver();
String url = kingbaseConfig.getJdbc();
Properties props = new Properties();
DeDriver deDriver = null;
if (StringUtils.isNotEmpty(kingbaseConfig.getAuthMethod()) && kingbaseConfig.getAuthMethod().equalsIgnoreCase("kerberos")) {
System.setProperty("java.security.krb5.conf", "/opt/dataease/conf/krb5.conf");
ExtendedJdbcClassLoader classLoader;
if (isDefaultClassLoader(customDriver)) {
classLoader = extendedJdbcClassLoader;
} else {
deDriver = deDriverMapper.selectByPrimaryKey(customDriver);
classLoader = getCustomJdbcClassLoader(deDriver);
}
Class<?> ConfigurationClass = classLoader.loadClass("org.apache.hadoop.conf.Configuration");
Method set = ConfigurationClass.getMethod("set", String.class, String.class);
Object obj = ConfigurationClass.newInstance();
set.invoke(obj, "hadoop.security.authentication", "Kerberos");
Class<?> UserGroupInformationClass = classLoader.loadClass("org.apache.hadoop.security" +
".UserGroupInformation");
Method setConfiguration = UserGroupInformationClass.getMethod("setConfiguration", ConfigurationClass);
Method loginUserFromKeytab = UserGroupInformationClass.getMethod("loginUserFromKeytab", String.class,
String.class);
setConfiguration.invoke(null, obj);
loginUserFromKeytab.invoke(null, kingbaseConfig.getUsername(),
"/opt/dataease/conf/" + kingbaseConfig.getPassword());
} else {
if (StringUtils.isNotBlank(kingbaseConfig.getUsername())) {
props.setProperty("user", kingbaseConfig.getUsername());
if (StringUtils.isNotBlank(kingbaseConfig.getPassword())) {
props.setProperty("password", kingbaseConfig.getPassword());
}
if (StringUtils.isNotBlank(username)) {
props.setProperty("user", username);
if (StringUtils.isNotBlank(password)) {
props.setProperty("password", password);
}
}
Connection conn;
String driverClassName;
ExtendedJdbcClassLoader jdbcClassLoader;
@ -147,24 +122,18 @@ public class KingbaseDsProvider extends DefaultJdbcProvider {
*/
@Override
public List<TableField> getTableFields(DatasourceRequest datasourceRequest) throws Exception {
List<TableField> list = new LinkedList<>();
try (Connection connection = getConnectionFromPool(datasourceRequest)) {
DatabaseMetaData databaseMetaData = connection.getMetaData();
ResultSet resultSet = databaseMetaData.getColumns(null, null, datasourceRequest.getTable(), "%");
String tableNamePattern = datasourceRequest.getTable();
String schemaPattern = "%";
ResultSet resultSet = databaseMetaData.getColumns(null, schemaPattern, tableNamePattern, "%");
while (resultSet.next()) {
String tableName = resultSet.getString("TABLE_NAME").toUpperCase();
String database;
database = resultSet.getString("TABLE_CAT");
if (database != null) {
if (tableName.equals(datasourceRequest.getTable()) && database.equalsIgnoreCase(getDatabase(datasourceRequest))) {
TableField tableField = getTableFiled(resultSet, datasourceRequest);
list.add(tableField);
}
} else {
if (tableName.equals(datasourceRequest.getTable())) {
TableField tableField = getTableFiled(resultSet, datasourceRequest);
list.add(tableField);
}
String tableName = resultSet.getString("TABLE_NAME");
if (tableName.equals(datasourceRequest.getTable())) {
TableField tableField = getTableFiled(resultSet, datasourceRequest);
list.add(tableField);
}
}
resultSet.close();
@ -189,6 +158,7 @@ public class KingbaseDsProvider extends DefaultJdbcProvider {
* 获取表字段
*/
private TableField getTableFiled(ResultSet resultSet, DatasourceRequest datasourceRequest) throws SQLException {
TableField tableField = new TableField();
String colName = resultSet.getString("COLUMN_NAME");
tableField.setFieldName(colName);
@ -206,16 +176,17 @@ public class KingbaseDsProvider extends DefaultJdbcProvider {
tableField.setFieldSize(50);
}
if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.hive.name()) && tableField.getFieldType().equalsIgnoreCase("BOOLEAN")) {
String size = resultSet.getString("COLUMN_SIZE");
if (size == null) {
tableField.setFieldSize(1);
} else {
String size = resultSet.getString("COLUMN_SIZE");
if (size == null) {
tableField.setFieldSize(1);
} else {
tableField.setFieldSize(Integer.valueOf(size));
}
tableField.setFieldSize(Integer.valueOf(size));
}
if (StringUtils.isNotEmpty(tableField.getFieldType()) && tableField.getFieldType().equalsIgnoreCase("DECIMAL")) {
tableField.setAccuracy(Integer.valueOf(resultSet.getString("DECIMAL_DIGITS")));
}
return tableField;
}
@ -224,6 +195,7 @@ public class KingbaseDsProvider extends DefaultJdbcProvider {
*/
@Override
public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
String queryStr = getTablesSql(datasourceRequest);
JdbcConfiguration jdbcConfiguration =
new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class);
@ -246,10 +218,7 @@ public class KingbaseDsProvider extends DefaultJdbcProvider {
if (StringUtils.isEmpty(kingbaseConfig.getSchema())) {
throw new Exception("Database schema is empty.");
}
/*return "select a.table_name, b.comments from all_tables a, user_tab_comments b where a.table_name = b
.table_name and owner=upper('OWNER') ".replaceAll("OWNER",
kingbaseConfig.getSchema());*/
return ("select table_name from all_tables where owner=upper('OWNER') ").replaceAll("OWNER",
return ("select tablename from pg_tables where schemaname = 'SCHEMA' ").replaceAll("SCHEMA",
kingbaseConfig.getSchema());
}
@ -258,7 +227,6 @@ public class KingbaseDsProvider extends DefaultJdbcProvider {
*/
@Override
public String getSchemaSql(DatasourceRequest datasourceRequest) {
return "select * from all_users";
return "SELECT nspname FROM pg_namespace";
}
}

50
extensions/dataease-extensions-datasource/kingbase/kingbase-backend/src/main/java/io/dataease/plugins/datasource/kingbase/query/KingbaseConstants.java
View File

@ -3,58 +3,30 @@ package io.dataease.plugins.datasource.kingbase.query;
import io.dataease.plugins.common.constants.datasource.SQLConstants;
import static io.dataease.plugins.common.constants.DatasourceTypes.oracle;
public class KingbaseConstants extends SQLConstants {
public static final String KEYWORD_TABLE = oracle.getKeywordPrefix() + "%s" + oracle.getKeywordSuffix();
public static final String KEYWORD_TABLE = "%s";
public static final String KEYWORD_FIX = "%s." + oracle.getKeywordPrefix() + "%s" + oracle.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + "%s";
public static final String ALIAS_FIX = oracle.getAliasPrefix() + "%s" + oracle.getAliasSuffix();
public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)";
public static final String DATE_FORMAT = "to_timestamp(%s,'%s')";
public static final String FROM_UNIXTIME = "FROM_UNIXTIME(%s,'%s')";
public static final String ALIAS_FIX = "%s";
public static final String UNIX_TIMESTAMP = "floor(extract(epoch from(( %s - timestamp '1970-01-01 00:00:00')" +
"*1000))) ";
public static final String DATE_FORMAT = "to_char(%s, '%s')";
public static final String STR_TO_DATE = "to_timestamp(%s, '%s')";
public static final String FROM_UNIXTIME = "to_timestamp(%s)";
public static final String CAST = "CAST(%s AS %s)";
public static final String DEFAULT_DATE_FORMAT = "YYYY-MM-DD HH24:MI:SS";
public static final String DEFAULT_INT_FORMAT = "DECIMAL(20,0)";
public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,8)";
public static final String DEFAULT_INT_FORMAT = "numeric(38,0)";
public static final String DEFAULT_FLOAT_FORMAT = "numeric(38,8)";
public static final String WHERE_VALUE_NULL = "(NULL,'')";
public static final String WHERE_VALUE_VALUE = "'%s'";
public static final String AGG_COUNT = "COUNT(*)";
public static final String AGG_FIELD = "%s(%s)";
public static final String WHERE_BETWEEN = "'%s' AND '%s'";
public static final String BRACKETS = "(%s)";
public static final String TO_NUMBER = "TO_NUMBER(%s)";
public static final String TO_DATE = "TO_DATE(%s,'%s')";
public static final String TO_CHAR = "TO_CHAR(%s,'%s')";
public static final String DEFAULT_START_DATE = "'1970-01-01 8:0:0'";
public static final String TO_MS = " * 24 * 60 * 60 * 100";
public static final String CALC_SUB = "%s - %s";
// public static final String GROUP_CONCAT = "vm_concat(%s)";
public static final String GROUP_CONCAT = "to_char(listagg(%s,',' ) within GROUP (order by (%s)))";
public static final String NAME = "oracle";
public static final String NAME = "pg";
}

898
extensions/dataease-extensions-datasource/kingbase/kingbase-backend/src/main/java/io/dataease/plugins/datasource/kingbase/query/KingbaseQueryProvider.java
View File

File diff suppressed because it is too large Load Diff

2
extensions/dataease-extensions-datasource/kingbase/kingbase-backend/src/main/java/io/dataease/plugins/datasource/kingbase/service/KingbaseService.java
View File

@ -53,7 +53,7 @@ public class KingbaseService extends DatasourceService {
@Override
public DataSourceType getDataSourceType() {
DataSourceType dataSourceType = new DataSourceType("kingbase", "KingBase", true, "",
DatasourceCalculationMode.DIRECT_AND_SYNC, true);
DatasourceCalculationMode.DIRECT, true);
dataSourceType.setKeywordPrefix("\"");
dataSourceType.setKeywordSuffix("\"");
dataSourceType.setAliasPrefix("\"");

1
sdk/dataease-plugin-common/src/main/java/io/dataease/plugins/common/constants/DatasourceTypes.java
View File

@ -13,6 +13,7 @@ public enum DatasourceTypes {
StarRocks("StarRocks", "StarRocks", "`", "`", "'", "'", "characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true", true, DatasourceCalculationMode.DIRECT_AND_SYNC, null,null, true, DatabaseClassification.OLAP),
ds_doris("ds_doris", "Doris", "`", "`", "'", "'", "characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true", true, DatasourceCalculationMode.DIRECT_AND_SYNC, null, null,true, DatabaseClassification.OLAP),
pg("pg", "PostgreSQL", "\"", "\"", "\"", "\"", "", true, DatasourceCalculationMode.DIRECT_AND_SYNC, null, null,true, DatabaseClassification.OLTP),
kingbase("kingbase", "KingBase", "\"", "\"", "\"", "\"", "", true, DatasourceCalculationMode.DIRECT, null,null,true, DatabaseClassification.OLTP),
sqlServer("sqlServer", "SQL Server", "\"", "\"", "\"", "\"", "", true, DatasourceCalculationMode.DIRECT_AND_SYNC, null, null,true, DatabaseClassification.OLTP),
oracle("oracle", "Oracle", "\"", "\"", "\"", "\"", "", true, DatasourceCalculationMode.DIRECT_AND_SYNC, Arrays.asList("Default", "GBK", "BIG5", "ISO-8859-1", "UTF-8", "UTF-16", "CP850", "EUC_JP", "EUC_KR"), Arrays.asList("Default", "GBK", "UTF-8"),true, DatabaseClassification.OLTP),
mongo("mongo", "MongoDB", "`", "`", "\"", "\"", "rebuildschema=true&authSource=admin", true, DatasourceCalculationMode.DIRECT, null, null,true, DatabaseClassification.OLTP),