center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix: 修复精简模式下同步数据

Browse Source
This commit is contained in:
taojinlong 2022-03-14 21:20:23 +08:00
parent 5afa737e36
commit 5eb54eab52
11 changed files with 121 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java
View File

@ -100,6 +100,8 @@ public class ExcelXlsxReader extends DefaultHandler {
*/
private String formatString;
//定义前一个元素和当前元素的位置，用来计算其中空的单元格数量，如A6和A8等
private String preRef = null, ref = null;

2
backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java
View File

@ -172,7 +172,7 @@ public class DataSetTableController {
@ApiOperation("检测doris")
@PostMapping("checkDorisTableIsExists/{id}")
public Boolean checkDorisTableIsExists(@PathVariable String id) throws Exception {
return dataSetTableService.checkDorisTableIsExists(id);
return dataSetTableService.checkEngineTableIsExists(id);
}
@ApiOperation("搜索")

6
backend/src/main/java/io/dataease/provider/DDLProviderImpl.java
View File

@ -39,9 +39,9 @@ public class DDLProviderImpl extends DDLProvider {
Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) {
values.append("(").append(Md5Utils.md5(String.join(",", Arrays.asList(strings))))
.append("," ).append(String.join(",", Arrays.asList(strings)))
.append("),");
values.append("('").append(Md5Utils.md5(String.join(",", Arrays.asList(strings))))
.append("','" ).append(String.join("','", Arrays.asList(strings)))
.append("'),");
}
return insertSql + values.substring(0, values.length() - 1);
}

5
backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
View File

@ -395,6 +395,7 @@ public class JdbcProvider extends DatasourceProvider {
dataSource = jdbcConnection.get(datasourceRequest.getDatasource().getId());
if (dataSource != null) {
dataSource.close();
jdbcConnection.remove(datasourceRequest.getDatasource().getId());
}
break;
default:
@ -426,6 +427,7 @@ public class JdbcProvider extends DatasourceProvider {
case mysql:
case mariadb:
case engine_doris:
case engine_mysql:
case ds_doris:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
username = mysqlConfiguration.getUsername();
@ -529,6 +531,7 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) {
case mysql:
case mariadb:
case engine_mysql:
case engine_doris:
case ds_doris:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
@ -604,6 +607,7 @@ public class JdbcProvider extends DatasourceProvider {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType());
switch (datasourceType) {
case mysql:
case engine_mysql:
case mariadb:
JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class);
return String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '%s' ;", jdbcConfiguration.getDataBase());
@ -657,6 +661,7 @@ public class JdbcProvider extends DatasourceProvider {
case mysql:
case mariadb:
case engine_doris:
case engine_mysql:
case ds_doris:
case ck:
return null;

2
backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java
View File

@ -25,7 +25,7 @@ public class MysqlConstants extends SQLConstants {
public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%S";
public static final String DEFAULT_INT_FORMAT = "BIGINT";
public static final String DEFAULT_INT_FORMAT = "DECIMAL(20,0)";
public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)";

62
backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java
View File

@ -1,15 +1,24 @@
package io.dataease.provider.engine.mysql;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.commons.utils.TableUtils;
import io.dataease.provider.DDLProviderImpl;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* @Author gin
* @Date 2021/5/17 4:27 下午
*/
@Service("mysqlEngineDDL")
public class MysqlDDLProvider extends DDLProviderImpl {
private static final String creatTableSql =
"CREATE TABLE IF NOT EXISTS `TABLE_NAME`" +
"Column_Fields;" ;
@Override
public String createView(String name, String viewSQL) {
return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
@ -27,9 +36,60 @@ public class MysqlDDLProvider extends DDLProviderImpl {
@Override
public String replaceTable(String name){
String replaceTableSql = "rename table FROM_TABLE to FROM_TABLE_tmp, TO_TABLE to FROM_TABLE, FROM_TABLE_tmp to TO_TABLE; "
String replaceTableSql = "rename table FROM_TABLE to FROM_TABLE_tmp, TO_TABLE to FROM_TABLE, FROM_TABLE_tmp to TO_TABLE"
.replace("FROM_TABLE", name).replace("TO_TABLE", TableUtils.tmpName(name));
String dropTableSql = "DROP TABLE IF EXISTS " + TableUtils.tmpName(name);
return replaceTableSql + ";" + dropTableSql;
}
@Override
public String createTableSql(String tableName, List<DatasetTableField> datasetTableFields) {
String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
return creatTableSql.replace("TABLE_NAME", tableName).replace("Column_Fields", dorisTableColumnSql);
}
private String createDorisTableColumnSql(final List<DatasetTableField> datasetTableFields) {
StringBuilder Column_Fields = new StringBuilder("dataease_uuid varchar(50), `");
for (DatasetTableField datasetTableField : datasetTableFields) {
Column_Fields.append(datasetTableField.getDataeaseName()).append("` ");
Integer size = datasetTableField.getSize() * 4;
switch (datasetTableField.getDeExtractType()) {
case 0:
if (size < 65533) {
Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
}else {
Column_Fields.append("longtext").append(",`");
}
break;
case 1:
size = size < 50? 50 : size;
if (size < 65533) {
Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
}else {
Column_Fields.append("longtext").append(",`");
}
break;
case 2:
Column_Fields.append("varchar(100)").append(",`");
break;
case 3:
Column_Fields.append("varchar(100)").append(",`");
break;
case 4:
Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
break;
default:
if (size < 65533) {
Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
}else {
Column_Fields.append("longtext").append(",`");
}
break;
}
}
Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 1)).append("PRIMARY KEY(dataease_uuid)");
Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
return Column_Fields.toString();
}
}

14
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -551,7 +551,7 @@ public class DataSetTableService {
}
} else {
// check doris table
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) {
if (!checkEngineTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
Datasource ds = engineService.getDeEngine();
@ -620,7 +620,7 @@ public class DataSetTableService {
}
} else {
// check doris table
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) {
if (!checkEngineTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
Datasource ds = engineService.getDeEngine();
@ -648,7 +648,7 @@ public class DataSetTableService {
}
}
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) {
if (!checkEngineTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
@ -2167,12 +2167,12 @@ public class DataSetTableService {
return filePath;
}
public Boolean checkDorisTableIsExists(String id) throws Exception {
Datasource dorisDatasource = engineService.getDeEngine();
public Boolean checkEngineTableIsExists(String id) throws Exception {
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
QueryProvider qp = ProviderFactory.getQueryProvider(dorisDatasource.getType());
datasourceRequest.setDatasource(engine);
QueryProvider qp = ProviderFactory.getQueryProvider(engine.getType());
datasourceRequest.setQuery(qp.searchTable(TableUtils.tableName(id)));
List<String[]> data = jdbcProvider.getData(datasourceRequest);
return CollectionUtils.isNotEmpty(data);

5
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -445,6 +445,7 @@ public class ExtractDataService {
List<String[]> dataList = result.get("dataList");
if (engineService.isSimpleMode()) {
extractDataForSimpleMode(extractType, datasetTable.getId(), dataList);
return;
}
Datasource engine = engineService.getDeEngine();
@ -712,7 +713,9 @@ public class ExtractDataService {
for (ExcelSheetData sheet : excelXlsxReader.totalSheets) {
if (sheet.getExcelLable().equalsIgnoreCase(excelSheetData.getExcelLable())) {
for (List<String> dataItem : sheet.getData()) {
data.add(dataItem.toArray(new String[dataItem.size()]));
if(dataItem.size()>0){
data.add(dataItem.toArray(new String[dataItem.size()]));
}
}
}
}

1
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@ -323,7 +323,6 @@ public class DatasourceService {
List<Datasource> datasources = datasourceMapper.selectByExampleWithBLOBs(new DatasourceExample());
datasources.forEach(datasource -> {
commonThreadPool.addTask(()->{
System.out.println(System.currentTimeMillis());
try {
handleConnectionPool(datasource, "add");
} catch (Exception e) {

40
backend/src/main/java/io/dataease/service/engine/EngineService.java
View File

@ -31,7 +31,7 @@ public class EngineService {
private DeEngineMapper deEngineMapper;
@Resource
private DatasourceService datasource;
static private Datasource ds = new Datasource();
static private Datasource ds = null;
public Boolean isLocalMode(){
@ -59,6 +59,9 @@ public class EngineService {
}
public ResultHolder validate(DatasourceDTO datasource) throws Exception {
if(StringUtils.isEmpty(datasource.getType()) || StringUtils.isEmpty(datasource.getConfiguration())){
throw new Exception("未完整设置数据引擎");
}
try {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
@ -78,13 +81,22 @@ public class EngineService {
deEngineMapper.updateByPrimaryKeyWithBLOBs(engine);
}
datasource.handleConnectionPool(this.ds, "delete");
BeanUtils.copyBean(this.ds, engine);
setDs(engine);
datasource.handleConnectionPool(this.ds, "add");
return ResultHolder.success(engine);
}
private void setDs(DeEngine engine){
if(this.ds == null){
this.ds = new Datasource();
BeanUtils.copyBean(this.ds, engine);
}else {
BeanUtils.copyBean(this.ds, engine);
}
}
public Datasource getDeEngine() throws Exception{
if (this.ds != null || StringUtils.isNotEmpty(ds.getType())) {
if (this.ds != null) {
return this.ds;
}
if(isLocalMode()){
@ -97,21 +109,23 @@ public class EngineService {
jsonObject.put("port", env.getProperty("doris.port", "9030"));
jsonObject.put("httpPort", env.getProperty("doris.httpPort", "8030"));
Datasource datasource = new Datasource();
datasource.setId("doris");
datasource.setName("doris");
datasource.setDesc("doris");
datasource.setType("engine_doris");
datasource.setConfiguration(jsonObject.toJSONString());
this.ds = datasource;
}
if(isSimpleMode()){
DeEngine engine = new DeEngine();
engine.setId("doris");
engine.setName("doris");
engine.setDesc("doris");
engine.setType("engine_doris");
engine.setConfiguration(jsonObject.toJSONString());
setDs(engine);
}else {
List<DeEngine> deEngines = deEngineMapper.selectByExampleWithBLOBs(new DeEngineExample());
if(CollectionUtils.isEmpty(deEngines)){
throw new Exception("未设置数据引擎");
}
BeanUtils.copyBean(this.ds, deEngines.get(0));
setDs(deEngines.get(0));
}
// if(isSimpleMode()){
//
// }
//TODO cluster mode
return this.ds;

19
frontend/src/views/system/SysParam/SimpleModeSetting.vue
View File

@ -101,7 +101,7 @@ export default {
return {
form:
{
type: 'mysql',
type: 'engine_mysql',
configuration: {
host: '',
dataBase: '',
@ -129,30 +129,31 @@ export default {
disabledSave: false,
loading: false,
rules: {
host: [
'configuration.host': [
{
required: true,
message: this.$t('system_parameter_setting.host'),
message: this.$t('datasource.please_input_host'),
trigger: ['change', 'blur']
}
],
port: [
'configuration.port': [
{
required: true,
message: this.$t('system_parameter_setting.port'),
message: this.$t('datasource.please_input_port'),
trigger: ['change', 'blur']
}
],
account: [
'configuration.dataBase': [
{
required: true,
message: this.$t('system_parameter_setting.account'),
message: this.$t('datasource.please_input_data_base'),
trigger: ['change', 'blur']
}]
}
]
},
allTypes: [
{
name: 'mysql',
name: 'engine_mysql',
label: 'MySQL',
type: 'jdbc',
extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'