center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix: 尝试修复支持数据库字符集

Browse Source
This commit is contained in:
taojinlong 2022-06-25 17:24:23 +08:00
parent be11424539
commit 91491eb1ab
7 changed files with 49 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/main/java/io/dataease/provider/ProviderFactory.java
View File

@ -27,6 +27,7 @@ public class ProviderFactory implements ApplicationContextAware {
DataSourceType dataSourceType = new DataSourceType(d.getType(), d.getName(), false, d.getExtraParams(), d.getCalculationMode(), d.isJdbc());
if(dataSourceType.getType().equalsIgnoreCase("oracle")){
dataSourceType.setCharset(d.getCharset());
dataSourceType.setTargetCharset(d.getTargetCharset());
}
beanFactory.registerSingleton(d.getType(), dataSourceType);
}

24
backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
View File

@ -216,10 +216,12 @@ public class JdbcProvider extends DefaultJdbcProvider {
private List<String[]> getDataResult(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
String charset = null;
String targetCharset = "UTF-8";
if (datasourceRequest != null && datasourceRequest.getDatasource().getType().equalsIgnoreCase("oracle")) {
JdbcConfiguration JdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class);
if (StringUtils.isNotEmpty(JdbcConfiguration.getCharset()) && !JdbcConfiguration.getCharset().equalsIgnoreCase("Default")) {
charset = JdbcConfiguration.getCharset();
JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class);
if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset()) && !jdbcConfiguration.getCharset().equalsIgnoreCase("Default")) {
charset = jdbcConfiguration.getCharset();
targetCharset = jdbcConfiguration.getTargetCharset();
}
}
List<String[]> list = new LinkedList<>();
@ -239,11 +241,17 @@ public class JdbcProvider extends DefaultJdbcProvider {
row[j] = rs.getBoolean(j + 1) ? "1" : "0";
break;
default:
if (charset != null && StringUtils.isNotEmpty(rs.getString(j + 1))) {
row[j] = new String(rs.getString(j + 1).getBytes(charset), "UTF-8");
if (metaData.getColumnTypeName(j + 1).toLowerCase().equalsIgnoreCase("blob")) {
row[j] = rs.getBlob(j + 1) == null ? "" : rs.getBlob(j + 1).toString();
} else {
row[j] = rs.getString(j + 1);
if (charset != null && StringUtils.isNotEmpty(rs.getString(j + 1))) {
String orginStr = new String(rs.getString(j + 1).getBytes(charset), targetCharset);
row[j] = new String(orginStr.getBytes("UTF-8"), "UTF-8");
} else {
row[j] = rs.getString(j + 1);
}
}
break;
}
}
@ -308,7 +316,7 @@ public class JdbcProvider extends DefaultJdbcProvider {
String queryStr = getTablesSql(datasourceRequest);
try (Connection con = getConnection(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryStr)) {
} catch (Exception e) {
LogUtil.error("Datasource is invalid: " + datasourceRequest.getDatasource().getName() , e);
LogUtil.error("Datasource is invalid: " + datasourceRequest.getDatasource().getName(), e);
io.dataease.plugins.common.exception.DataEaseException.throwException(e.getMessage());
}
return "Success";
@ -452,7 +460,7 @@ public class JdbcProvider extends DefaultJdbcProvider {
driverClassName = defaultDriver;
jdbcClassLoader = extendedJdbcClassLoader;
} else {
if(deDriver == null){
if (deDriver == null) {
deDriver = deDriverMapper.selectByPrimaryKey(customDriver);
}
driverClassName = deDriver.getDriverClass();

26
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -13,6 +13,7 @@ import io.dataease.plugins.common.constants.DatasourceTypes;
import io.dataease.plugins.common.constants.DeTypeConstants;
import io.dataease.plugins.common.dto.datasource.TableField;
import io.dataease.plugins.common.request.datasource.DatasourceRequest;
import io.dataease.plugins.datasource.entity.JdbcConfiguration;
import io.dataease.plugins.datasource.provider.Provider;
import io.dataease.plugins.datasource.query.QueryProvider;
import io.dataease.provider.DDLProvider;
@ -919,7 +920,7 @@ public class ExtractDataService {
}
transMeta.addDatabase(dataMeta);
selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
inputStep = inputStep(transMeta, selectSQL);
inputStep = inputStep(transMeta, selectSQL, mysqlConfiguration);
udjcStep = udjc(datasetTableFields, DatasourceTypes.mysql);
break;
case sqlServer:
@ -927,7 +928,7 @@ public class ExtractDataService {
dataMeta = new DatabaseMeta("db", "MSSQLNATIVE", "Native", sqlServerConfiguration.getHost().trim(), sqlServerConfiguration.getDataBase(), sqlServerConfiguration.getPort().toString(), sqlServerConfiguration.getUsername(), sqlServerConfiguration.getPassword());
transMeta.addDatabase(dataMeta);
selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
inputStep = inputStep(transMeta, selectSQL);
inputStep = inputStep(transMeta, selectSQL, sqlServerConfiguration);
udjcStep = udjc(datasetTableFields, DatasourceTypes.sqlServer);
break;
case pg:
@ -935,7 +936,7 @@ public class ExtractDataService {
dataMeta = new DatabaseMeta("db", "POSTGRESQL", "Native", pgConfiguration.getHost().trim(), pgConfiguration.getDataBase(), pgConfiguration.getPort().toString(), pgConfiguration.getUsername(), pgConfiguration.getPassword());
transMeta.addDatabase(dataMeta);
selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
inputStep = inputStep(transMeta, selectSQL);
inputStep = inputStep(transMeta, selectSQL, pgConfiguration);
udjcStep = udjc(datasetTableFields, DatasourceTypes.pg);
break;
case oracle:
@ -947,9 +948,8 @@ public class ExtractDataService {
dataMeta = new DatabaseMeta("db", "ORACLE", "Native", oracleConfiguration.getHost().trim(), oracleConfiguration.getDataBase(), oracleConfiguration.getPort().toString(), oracleConfiguration.getUsername(), oracleConfiguration.getPassword());
}
transMeta.addDatabase(dataMeta);
selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
inputStep = inputStep(transMeta, selectSQL);
inputStep = inputStep(transMeta, selectSQL, oracleConfiguration);
udjcStep = udjc(datasetTableFields, DatasourceTypes.oracle);
break;
case ck:
@ -958,7 +958,7 @@ public class ExtractDataService {
dataMeta.setDatabaseType("Clickhouse");
transMeta.addDatabase(dataMeta);
selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
inputStep = inputStep(transMeta, selectSQL);
inputStep = inputStep(transMeta, selectSQL, chConfiguration);
udjcStep = udjc(datasetTableFields, DatasourceTypes.ck);
break;
case db2:
@ -967,7 +967,7 @@ public class ExtractDataService {
dataMeta.setDatabaseType("DB2");
transMeta.addDatabase(dataMeta);
selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
inputStep = inputStep(transMeta, selectSQL);
inputStep = inputStep(transMeta, selectSQL, db2Configuration);
udjcStep = udjc(datasetTableFields, DatasourceTypes.db2);
break;
case excel:
@ -1031,11 +1031,14 @@ public class ExtractDataService {
return selectSQL;
}
private StepMeta inputStep(TransMeta transMeta, String selectSQL) {
private StepMeta inputStep(TransMeta transMeta, String selectSQL, JdbcConfiguration jdbcConfiguration) {
TableInputMeta tableInput = new TableInputMeta();
DatabaseMeta database = transMeta.findDatabase(DatasetType.DB.name());
tableInput.setDatabaseMeta(database);
tableInput.setSQL(selectSQL);
if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset()) && !jdbcConfiguration.getCharset().equalsIgnoreCase("Default")) {
tableInput.setLazyConversionActive(true);
}
StepMeta fromStep = new StepMeta("TableInput", "Data Input", tableInput);
fromStep.setDraw(true);
fromStep.setLocation(100, 100);
@ -1095,7 +1098,12 @@ public class ExtractDataService {
private StepMeta outputStep(String dorisOutputTable, List<DatasetTableField> datasetTableFields, Datasource datasource) {
TextFileOutputMeta textFileOutputMeta = new TextFileOutputMeta();
textFileOutputMeta.setEncoding("UTF-8");
JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasource.getConfiguration(), JdbcConfiguration.class);
if (StringUtils.isNotEmpty(jdbcConfiguration.getTargetCharset()) && !jdbcConfiguration.getTargetCharset().equalsIgnoreCase("Default")) {
textFileOutputMeta.setEncoding(jdbcConfiguration.getTargetCharset());
}else {
textFileOutputMeta.setEncoding("UTF-8");
}
textFileOutputMeta.setHeaderEnabled(false);
textFileOutputMeta.setFilename(root_path + dorisOutputTable);
textFileOutputMeta.setSeparator(separator);

4
frontend/src/lang/en.js
View File

@ -1404,7 +1404,9 @@ export default {
schema: 'Database Schema',
please_choose_schema: 'Please select Schema',
charset: 'Charset',
please_choose_charset: 'Please select Charset',
targetCharset: 'Target Charset',
please_choose_targetCharset: 'Please select target charset',
please_choose_charset: 'Please select charset',
edit_datasource_msg: 'Modifying the data source information may make the data set under the modified data source unavailable. Confirm the modification？',
repeat_datasource_msg: 'Data source information with the same configuration already exists, ',
confirm_save: 'Confirm save?',

2
frontend/src/lang/tw.js
View File

@ -1405,6 +1405,8 @@ export default {
schema: '數據庫 Schema',
please_choose_schema: '請選擇數據庫 Schema',
charset: '字符集',
targetCharset: '目標字符集',
please_choose_targetCharset: '請選擇目標字符集',
please_choose_charset: '請選擇數據庫字符集',
edit_datasource_msg: '修改數據源信息，可能會導致改數據源下的數據集不可用，確認修改？',
repeat_datasource_msg: '已經存在相同配置的數據源信息，',

2
frontend/src/lang/zh.js
View File

@ -1410,8 +1410,10 @@ export default {
get_schema: '获取 Schema',
schema: '数据库 Schema',
charset: '字符集',
targetCharset: '目标字符集',
please_choose_schema: '请选择数据库 Schema',
please_choose_charset: '请选择数据库字符集',
please_choose_targetCharset: '请选择目标字符集',
edit_datasource_msg: '修改数据源信息，可能会导致该数据源下的数据集不可用，确认修改？',
repeat_datasource_msg: '已经存在相同配置的数据源信息, ',
confirm_save: '确认保存?',

8
frontend/src/views/system/datasource/DsConfiguration.vue
View File

@ -228,6 +228,14 @@
</el-select>
</el-form-item>
<el-form-item v-if="form.type=='oracle'" :label="$t('datasource.charset')">
<el-select v-model="form.configuration.targetCharset" filterable
:placeholder="$t('datasource.please_choose_targetCharset')"
class="select-width">
<el-option v-for="item in datasourceType.targetCharset" :key="item" :label="item" :value="item"/>
</el-select>
</el-form-item>
<el-collapse v-if="form.type !=='es' && form.type !== 'api' && form.type !== 'mongo'">
<el-collapse-item :title="$t('datasource.priority')" name="1">
<el-form-item :label="$t('datasource.initial_pool_size')" prop="configuration.initialPoolSize">