From 5eb54eab5229bf53d850b7edd470aa11b1cdf2b1 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Mon, 14 Mar 2022 21:20:23 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E7=B2=BE=E7=AE=80?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E4=B8=8B=E5=90=8C=E6=AD=A5=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commons/utils/ExcelXlsxReader.java | 2 + .../dataset/DataSetTableController.java | 2 +- .../io/dataease/provider/DDLProviderImpl.java | 6 +- .../provider/datasource/JdbcProvider.java | 5 ++ .../provider/engine/mysql/MysqlConstants.java | 2 +- .../engine/mysql/MysqlDDLProvider.java | 62 ++++++++++++++++++- .../service/dataset/DataSetTableService.java | 14 ++--- .../service/dataset/ExtractDataService.java | 5 +- .../service/datasource/DatasourceService.java | 1 - .../service/engine/EngineService.java | 40 ++++++++---- .../system/SysParam/SimpleModeSetting.vue | 19 +++--- 11 files changed, 121 insertions(+), 37 deletions(-) diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java index d381e754c7..dda075e2c7 100644 --- a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java @@ -100,6 +100,8 @@ public class ExcelXlsxReader extends DefaultHandler { */ private String formatString; + + //定义前一个元素和当前元素的位置，用来计算其中空的单元格数量，如A6和A8等 private String preRef = null, ref = null; diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java index 940cb645e9..e3ed90b204 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java @@ -172,7 +172,7 @@ public class DataSetTableController { @ApiOperation("检测doris") @PostMapping("checkDorisTableIsExists/{id}") public Boolean checkDorisTableIsExists(@PathVariable String id) throws Exception { - return dataSetTableService.checkDorisTableIsExists(id); + return dataSetTableService.checkEngineTableIsExists(id); } @ApiOperation("搜索") diff --git a/backend/src/main/java/io/dataease/provider/DDLProviderImpl.java b/backend/src/main/java/io/dataease/provider/DDLProviderImpl.java index ada0dd0294..f90781fcb3 100644 --- a/backend/src/main/java/io/dataease/provider/DDLProviderImpl.java +++ b/backend/src/main/java/io/dataease/provider/DDLProviderImpl.java @@ -39,9 +39,9 @@ public class DDLProviderImpl extends DDLProvider { Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size(); for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) { - values.append("(").append(Md5Utils.md5(String.join(",", Arrays.asList(strings)))) - .append("," ).append(String.join(",", Arrays.asList(strings))) - .append("),"); + values.append("('").append(Md5Utils.md5(String.join(",", Arrays.asList(strings)))) + .append("','" ).append(String.join("','", Arrays.asList(strings))) + .append("'),"); } return insertSql + values.substring(0, values.length() - 1); } diff --git a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java index d135676871..2a98a6c683 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java @@ -395,6 +395,7 @@ public class JdbcProvider extends DatasourceProvider { dataSource = jdbcConnection.get(datasourceRequest.getDatasource().getId()); if (dataSource != null) { dataSource.close(); + jdbcConnection.remove(datasourceRequest.getDatasource().getId()); } break; default: @@ -426,6 +427,7 @@ public class JdbcProvider extends DatasourceProvider { case mysql: case mariadb: case engine_doris: + case engine_mysql: case ds_doris: MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class); username = mysqlConfiguration.getUsername(); @@ -529,6 +531,7 @@ public class JdbcProvider extends DatasourceProvider { switch (datasourceType) { case mysql: case mariadb: + case engine_mysql: case engine_doris: case ds_doris: MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class); @@ -604,6 +607,7 @@ public class JdbcProvider extends DatasourceProvider { DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType()); switch (datasourceType) { case mysql: + case engine_mysql: case mariadb: JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class); return String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '%s' ;", jdbcConfiguration.getDataBase()); @@ -657,6 +661,7 @@ public class JdbcProvider extends DatasourceProvider { case mysql: case mariadb: case engine_doris: + case engine_mysql: case ds_doris: case ck: return null; diff --git a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java index d2590b85f8..e75a357da7 100644 --- a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java +++ b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java @@ -25,7 +25,7 @@ public class MysqlConstants extends SQLConstants { public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%S"; - public static final String DEFAULT_INT_FORMAT = "BIGINT"; + public static final String DEFAULT_INT_FORMAT = "DECIMAL(20,0)"; public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)"; diff --git a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java index 61e7168d26..91ba3ccf97 100644 --- a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java +++ b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java @@ -1,15 +1,24 @@ package io.dataease.provider.engine.mysql; +import io.dataease.base.domain.DatasetTableField; import io.dataease.commons.utils.TableUtils; import io.dataease.provider.DDLProviderImpl; import org.springframework.stereotype.Service; +import java.util.List; + /** * @Author gin * @Date 2021/5/17 4:27 下午 */ @Service("mysqlEngineDDL") public class MysqlDDLProvider extends DDLProviderImpl { + + private static final String creatTableSql = + "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" + + "Column_Fields;" ; + + @Override public String createView(String name, String viewSQL) { return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")"; @@ -27,9 +36,60 @@ public class MysqlDDLProvider extends DDLProviderImpl { @Override public String replaceTable(String name){ - String replaceTableSql = "rename table FROM_TABLE to FROM_TABLE_tmp, TO_TABLE to FROM_TABLE, FROM_TABLE_tmp to TO_TABLE; " + String replaceTableSql = "rename table FROM_TABLE to FROM_TABLE_tmp, TO_TABLE to FROM_TABLE, FROM_TABLE_tmp to TO_TABLE" .replace("FROM_TABLE", name).replace("TO_TABLE", TableUtils.tmpName(name)); String dropTableSql = "DROP TABLE IF EXISTS " + TableUtils.tmpName(name); return replaceTableSql + ";" + dropTableSql; } + + @Override + public String createTableSql(String tableName, List datasetTableFields) { + String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields); + return creatTableSql.replace("TABLE_NAME", tableName).replace("Column_Fields", dorisTableColumnSql); + } + + private String createDorisTableColumnSql(final List datasetTableFields) { + StringBuilder Column_Fields = new StringBuilder("dataease_uuid varchar(50), `"); + for (DatasetTableField datasetTableField : datasetTableFields) { + Column_Fields.append(datasetTableField.getDataeaseName()).append("` "); + Integer size = datasetTableField.getSize() * 4; + switch (datasetTableField.getDeExtractType()) { + case 0: + if (size < 65533) { + Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`"); + }else { + Column_Fields.append("longtext").append(",`"); + } + break; + case 1: + size = size < 50? 50 : size; + if (size < 65533) { + Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`"); + }else { + Column_Fields.append("longtext").append(",`"); + } + break; + case 2: + Column_Fields.append("varchar(100)").append(",`"); + break; + case 3: + Column_Fields.append("varchar(100)").append(",`"); + break; + case 4: + Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`"); + break; + default: + if (size < 65533) { + Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`"); + }else { + Column_Fields.append("longtext").append(",`"); + } + break; + } + } + + Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 1)).append("PRIMARY KEY(dataease_uuid)"); + Column_Fields = new StringBuilder("(" + Column_Fields + ")\n"); + return Column_Fields.toString(); + } } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 64ade01908..67b2d1be13 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -551,7 +551,7 @@ public class DataSetTableService { } } else { // check doris table - if (!checkDorisTableIsExists(dataSetTableRequest.getId())) { + if (!checkEngineTableIsExists(dataSetTableRequest.getId())) { throw new RuntimeException(Translator.get("i18n_data_not_sync")); } Datasource ds = engineService.getDeEngine(); @@ -620,7 +620,7 @@ public class DataSetTableService { } } else { // check doris table - if (!checkDorisTableIsExists(dataSetTableRequest.getId())) { + if (!checkEngineTableIsExists(dataSetTableRequest.getId())) { throw new RuntimeException(Translator.get("i18n_data_not_sync")); } Datasource ds = engineService.getDeEngine(); @@ -648,7 +648,7 @@ public class DataSetTableService { } } } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { - if (!checkDorisTableIsExists(dataSetTableRequest.getId())) { + if (!checkEngineTableIsExists(dataSetTableRequest.getId())) { throw new RuntimeException(Translator.get("i18n_data_not_sync")); } @@ -2167,12 +2167,12 @@ public class DataSetTableService { return filePath; } - public Boolean checkDorisTableIsExists(String id) throws Exception { - Datasource dorisDatasource = engineService.getDeEngine(); + public Boolean checkEngineTableIsExists(String id) throws Exception { + Datasource engine = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); - datasourceRequest.setDatasource(dorisDatasource); - QueryProvider qp = ProviderFactory.getQueryProvider(dorisDatasource.getType()); + datasourceRequest.setDatasource(engine); + QueryProvider qp = ProviderFactory.getQueryProvider(engine.getType()); datasourceRequest.setQuery(qp.searchTable(TableUtils.tableName(id))); List data = jdbcProvider.getData(datasourceRequest); return CollectionUtils.isNotEmpty(data); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 9795f86261..e91982aa8d 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -445,6 +445,7 @@ public class ExtractDataService { List dataList = result.get("dataList"); if (engineService.isSimpleMode()) { extractDataForSimpleMode(extractType, datasetTable.getId(), dataList); + return; } Datasource engine = engineService.getDeEngine(); @@ -712,7 +713,9 @@ public class ExtractDataService { for (ExcelSheetData sheet : excelXlsxReader.totalSheets) { if (sheet.getExcelLable().equalsIgnoreCase(excelSheetData.getExcelLable())) { for (List dataItem : sheet.getData()) { - data.add(dataItem.toArray(new String[dataItem.size()])); + if(dataItem.size()>0){ + data.add(dataItem.toArray(new String[dataItem.size()])); + } } } } diff --git a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java index 0f3b597bcc..73e91e0ae1 100644 --- a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java +++ b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java @@ -323,7 +323,6 @@ public class DatasourceService { List datasources = datasourceMapper.selectByExampleWithBLOBs(new DatasourceExample()); datasources.forEach(datasource -> { commonThreadPool.addTask(()->{ - System.out.println(System.currentTimeMillis()); try { handleConnectionPool(datasource, "add"); } catch (Exception e) { diff --git a/backend/src/main/java/io/dataease/service/engine/EngineService.java b/backend/src/main/java/io/dataease/service/engine/EngineService.java index 627c67e311..181d1acc6b 100644 --- a/backend/src/main/java/io/dataease/service/engine/EngineService.java +++ b/backend/src/main/java/io/dataease/service/engine/EngineService.java @@ -31,7 +31,7 @@ public class EngineService { private DeEngineMapper deEngineMapper; @Resource private DatasourceService datasource; - static private Datasource ds = new Datasource(); + static private Datasource ds = null; public Boolean isLocalMode(){ @@ -59,6 +59,9 @@ public class EngineService { } public ResultHolder validate(DatasourceDTO datasource) throws Exception { + if(StringUtils.isEmpty(datasource.getType()) || StringUtils.isEmpty(datasource.getConfiguration())){ + throw new Exception("未完整设置数据引擎"); + } try { DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); @@ -78,13 +81,22 @@ public class EngineService { deEngineMapper.updateByPrimaryKeyWithBLOBs(engine); } datasource.handleConnectionPool(this.ds, "delete"); - BeanUtils.copyBean(this.ds, engine); + setDs(engine); datasource.handleConnectionPool(this.ds, "add"); return ResultHolder.success(engine); } + private void setDs(DeEngine engine){ + if(this.ds == null){ + this.ds = new Datasource(); + BeanUtils.copyBean(this.ds, engine); + }else { + BeanUtils.copyBean(this.ds, engine); + } + } + public Datasource getDeEngine() throws Exception{ - if (this.ds != null || StringUtils.isNotEmpty(ds.getType())) { + if (this.ds != null) { return this.ds; } if(isLocalMode()){ @@ -97,21 +109,23 @@ public class EngineService { jsonObject.put("port", env.getProperty("doris.port", "9030")); jsonObject.put("httpPort", env.getProperty("doris.httpPort", "8030")); - Datasource datasource = new Datasource(); - datasource.setId("doris"); - datasource.setName("doris"); - datasource.setDesc("doris"); - datasource.setType("engine_doris"); - datasource.setConfiguration(jsonObject.toJSONString()); - this.ds = datasource; - } - if(isSimpleMode()){ + DeEngine engine = new DeEngine(); + engine.setId("doris"); + engine.setName("doris"); + engine.setDesc("doris"); + engine.setType("engine_doris"); + engine.setConfiguration(jsonObject.toJSONString()); + setDs(engine); + }else { List deEngines = deEngineMapper.selectByExampleWithBLOBs(new DeEngineExample()); if(CollectionUtils.isEmpty(deEngines)){ throw new Exception("未设置数据引擎"); } - BeanUtils.copyBean(this.ds, deEngines.get(0)); + setDs(deEngines.get(0)); } +// if(isSimpleMode()){ +// +// } //TODO cluster mode return this.ds; diff --git a/frontend/src/views/system/SysParam/SimpleModeSetting.vue b/frontend/src/views/system/SysParam/SimpleModeSetting.vue index 5ef33b3edd..6871b83ac5 100644 --- a/frontend/src/views/system/SysParam/SimpleModeSetting.vue +++ b/frontend/src/views/system/SysParam/SimpleModeSetting.vue @@ -101,7 +101,7 @@ export default { return { form: { - type: 'mysql', + type: 'engine_mysql', configuration: { host: '', dataBase: '', @@ -129,30 +129,31 @@ export default { disabledSave: false, loading: false, rules: { - host: [ + 'configuration.host': [ { required: true, - message: this.$t('system_parameter_setting.host'), + message: this.$t('datasource.please_input_host'), trigger: ['change', 'blur'] } ], - port: [ + 'configuration.port': [ { required: true, - message: this.$t('system_parameter_setting.port'), + message: this.$t('datasource.please_input_port'), trigger: ['change', 'blur'] } ], - account: [ + 'configuration.dataBase': [ { required: true, - message: this.$t('system_parameter_setting.account'), + message: this.$t('datasource.please_input_data_base'), trigger: ['change', 'blur'] - }] + } + ] }, allTypes: [ { - name: 'mysql', + name: 'engine_mysql', label: 'MySQL', type: 'jdbc', extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true' From 65aaff16a14ddeb14fa8ea50f0c2137b6a13fb09 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Mon, 14 Mar 2022 21:30:22 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=20mongoDB=20?= =?UTF-8?q?=E7=9A=84=E8=BF=9E=E6=8E=A5=E5=AD=97=E7=AC=A6=E4=B8=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/system/datasource/form.vue | 286 +++++++++--------- 1 file changed, 143 insertions(+), 143 deletions(-) diff --git a/frontend/src/views/system/datasource/form.vue b/frontend/src/views/system/datasource/form.vue index 1e60385f90..721f1a805f 100644 --- a/frontend/src/views/system/datasource/form.vue +++ b/frontend/src/views/system/datasource/form.vue @@ -45,9 +45,12 @@ - - - + + +