diff --git a/backend/pom.xml b/backend/pom.xml index 092853654e..1a0ad78d0b 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -358,6 +358,13 @@ dataease-plugin-xpack 1.0 + + + com.oracle.database.jdbc + ojdbc8 + 12.2.0.1 + + diff --git a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java index 51df39c55e..2d77ec9549 100644 --- a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java +++ b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java @@ -1,5 +1,5 @@ package io.dataease.datasource.constants; public enum DatasourceTypes { - mysql, sqlServer, excel, doris + mysql, sqlServer, excel, doris, oracle } diff --git a/backend/src/main/java/io/dataease/datasource/dto/OracleConfigration.java b/backend/src/main/java/io/dataease/datasource/dto/OracleConfigration.java new file mode 100644 index 0000000000..6323791f90 --- /dev/null +++ b/backend/src/main/java/io/dataease/datasource/dto/OracleConfigration.java @@ -0,0 +1,27 @@ +package io.dataease.datasource.dto; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class OracleConfigration extends JdbcDTO { + + private String driver = "oracle.jdbc.driver.OracleDriver"; + private String connectionType; + + public String getJdbc() { + // 连接参数先写死，后边要把编码、时区等参数放到数据源的设置中 + if(getConnectionType().equalsIgnoreCase("serviceName")){ + return "jdbc:oracle:thin:@HOSTNAME:PORT/DATABASE" + .replace("HOSTNAME", getHost()) + .replace("PORT", getPort().toString()) + .replace("DATABASE", getDataBase()); + }else { + return "jdbc:oracle:thin:@HOSTNAME:PORT:DATABASE" + .replace("HOSTNAME", getHost()) + .replace("PORT", getPort().toString()) + .replace("DATABASE", getDataBase()); + } + } +} diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java index e2d759c838..04a87ebde9 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java @@ -4,6 +4,7 @@ import com.google.gson.Gson; import com.mchange.v2.c3p0.ComboPooledDataSource; import io.dataease.datasource.constants.DatasourceTypes; import io.dataease.datasource.dto.MysqlConfigration; +import io.dataease.datasource.dto.OracleConfigration; import io.dataease.datasource.dto.SqlServerConfigration; import io.dataease.datasource.dto.TableFiled; import io.dataease.datasource.request.DatasourceRequest; @@ -19,8 +20,8 @@ import java.util.*; public class JdbcProvider extends DatasourceProvider { private static Map jdbcConnection = new HashMap<>(); - private static int initPoolSize = 5; - private static int maxConnections = 200; + private static int initPoolSize = 1; + private static int maxConnections = 1; @Override public List getData(DatasourceRequest datasourceRequest) throws Exception { @@ -207,22 +208,16 @@ public class JdbcProvider extends DatasourceProvider { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); String database = resultSet.getString("TABLE_CAT"); - if (tableName.equals(datasourceRequest.getTable()) && database.equalsIgnoreCase(getDatabase(datasourceRequest))) { - TableFiled tableFiled = new TableFiled(); - String colName = resultSet.getString("COLUMN_NAME"); - tableFiled.setFieldName(colName); - String remarks = resultSet.getString("REMARKS"); - if (remarks == null || remarks.equals("")) { - remarks = colName; + if(database != null){ + if (tableName.equals(datasourceRequest.getTable()) && database.equalsIgnoreCase(getDatabase(datasourceRequest))) { + TableFiled tableFiled = getTableFiled(resultSet); + list.add(tableFiled); } - tableFiled.setRemarks(remarks); - tableFiled.setFieldSize(Integer.valueOf(resultSet.getString("COLUMN_SIZE"))); - String dbType = resultSet.getString("TYPE_NAME"); - tableFiled.setFieldType(dbType); - if(StringUtils.isNotEmpty(dbType) && dbType.toLowerCase().contains("date") && tableFiled.getFieldSize() < 50 ){ - tableFiled.setFieldSize(50); + }else { + if (tableName.equals(datasourceRequest.getTable())) { + TableFiled tableFiled = getTableFiled(resultSet); + list.add(tableFiled); } - list.add(tableFiled); } } resultSet.close(); @@ -238,6 +233,24 @@ public class JdbcProvider extends DatasourceProvider { return list; } + private TableFiled getTableFiled(ResultSet resultSet) throws SQLException { + TableFiled tableFiled = new TableFiled(); + String colName = resultSet.getString("COLUMN_NAME"); + tableFiled.setFieldName(colName); + String remarks = resultSet.getString("REMARKS"); + if (remarks == null || remarks.equals("")) { + remarks = colName; + } + tableFiled.setRemarks(remarks); + tableFiled.setFieldSize(Integer.valueOf(resultSet.getString("COLUMN_SIZE"))); + String dbType = resultSet.getString("TYPE_NAME"); + tableFiled.setFieldType(dbType); + if(StringUtils.isNotEmpty(dbType) && dbType.toLowerCase().contains("date") && tableFiled.getFieldSize() < 50 ){ + tableFiled.setFieldSize(50); + } + return tableFiled; + } + @Override public void test(DatasourceRequest datasourceRequest) throws Exception { String queryStr = getTablesSql(datasourceRequest); @@ -320,7 +333,7 @@ public class JdbcProvider extends DatasourceProvider { dataSource.setTestConnectionOnCheckout(false); // 在每个connection 提交是校验有效性 dataSource.setTestConnectionOnCheckin(true); // 取得连接的同时将校验连接的有效性 dataSource.setCheckoutTimeout(60000); // 从连接池获取连接的超时时间，如设为0则无限期等待。单位毫秒，默认为0 - dataSource.setPreferredTestQuery("SELECT 1"); +// dataSource.setPreferredTestQuery("SELECT 1"); dataSource.setDebugUnreturnedConnectionStackTraces(true); dataSource.setUnreturnedConnectionTimeout(3600); jdbcConnection.put(datasourceRequest.getDatasource().getId(), dataSource); @@ -354,6 +367,13 @@ public class JdbcProvider extends DatasourceProvider { driver = sqlServerConfigration.getDriver(); jdbcurl = sqlServerConfigration.getJdbc(); break; + case oracle: + OracleConfigration oracleConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), OracleConfigration.class); + username = oracleConfigration.getUsername(); + password = oracleConfigration.getPassword(); + driver = oracleConfigration.getDriver(); + jdbcurl = oracleConfigration.getJdbc(); + break; default: break; } @@ -392,6 +412,13 @@ public class JdbcProvider extends DatasourceProvider { dataSource.setPassword(sqlServerConfigration.getPassword()); dataSource.setJdbcUrl(sqlServerConfigration.getJdbc()); break; + case oracle: + OracleConfigration oracleConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), OracleConfigration.class); + dataSource.setUser(oracleConfigration.getUsername()); + dataSource.setDriverClass(oracleConfigration.getDriver()); + dataSource.setPassword(oracleConfigration.getPassword()); + dataSource.setJdbcUrl(oracleConfigration.getJdbc()); + break; default: break; } @@ -424,6 +451,8 @@ public class JdbcProvider extends DatasourceProvider { case sqlServer: SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfigration.class); return "SELECT TABLE_NAME FROM DATABASE.INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE';".replace("DATABASE", sqlServerConfigration.getDataBase()); + case oracle: + return "select TABLE_NAME from USER_TABLES"; default: return "show tables;"; } diff --git a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java index bc609b2e75..88a720571a 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java +++ b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java @@ -42,6 +42,8 @@ public class ProviderFactory implements ApplicationContextAware { return context.getBean("dorisQuery", QueryProvider.class); case sqlServer: return context.getBean("sqlserverQuery", QueryProvider.class); + case oracle: + return context.getBean("oracleQuery", QueryProvider.class); default: return context.getBean("mysqlQuery", QueryProvider.class); } @@ -54,6 +56,8 @@ public class ProviderFactory implements ApplicationContextAware { return context.getBean("mysqlDDL", DDLProvider.class); case doris: return context.getBean("dorisDDL", DDLProvider.class); + case oracle: + return context.getBean("oracleDDL", DDLProvider.class); case sqlServer: return context.getBean("mysqlDDL", DDLProvider.class); default: diff --git a/backend/src/main/java/io/dataease/provider/oracle/OracleDDLProvider.java b/backend/src/main/java/io/dataease/provider/oracle/OracleDDLProvider.java new file mode 100644 index 0000000000..f6e1431801 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/oracle/OracleDDLProvider.java @@ -0,0 +1,26 @@ +package io.dataease.provider.oracle; + +import io.dataease.provider.DDLProvider; +import org.springframework.stereotype.Service; + +/** + * @Author gin + * @Date 2021/5/17 4:27 下午 + */ +@Service("oracleDDL") +public class OracleDDLProvider extends DDLProvider { + @Override + public String createView(String name, String viewSQL) { + return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")"; + } + + @Override + public String dropTable(String name) { + return "DROP TABLE IF EXISTS " + name; + } + + @Override + public String dropView(String name) { + return "DROP VIEW IF EXISTS " + name; + } +} diff --git a/backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java b/backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java new file mode 100644 index 0000000000..bf8c0e4dca --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java @@ -0,0 +1,472 @@ +package io.dataease.provider.oracle; + +import io.dataease.base.domain.DatasetTableField; +import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.dto.chart.ChartCustomFilterDTO; +import io.dataease.dto.chart.ChartViewFieldDTO; +import io.dataease.provider.QueryProvider; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +/** + * @Author gin + * @Date 2021/5/17 2:43 下午 + */ +@Service("oracleQuery") +public class OracleQueryProvider extends QueryProvider { + @Override + public Integer transFieldType(String field) { + switch (field) { + case "CHAR": + case "VARCHAR": + case "TEXT": + case "TINYTEXT": + case "MEDIUMTEXT": + case "LONGTEXT": + case "ENUM": + return 0;// 文本 + case "DATE": + case "TIME": + case "YEAR": + case "DATETIME": + case "TIMESTAMP": + return 1;// 时间 + case "INT": + case "SMALLINT": + case "MEDIUMINT": + case "INTEGER": + case "BIGINT": + return 2;// 整型 + case "FLOAT": + case "DOUBLE": + case "DECIMAL": + return 3;// 浮点 + case "BIT": + case "TINYINT": + return 4;// 布尔 + default: + return 0; + } + } + + @Override + public String createQueryCountSQL(String table) { + return MessageFormat.format("SELECT COUNT(*) FROM {0}", table); + } + + @Override + public String createQueryCountSQLAsTmp(String sql) { + return createQueryCountSQL(" (" + sqlFix(sql) + ") AS tmp "); + } + + @Override + public String createSQLPreview(String sql, String orderBy) { + return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp ORDER BY null " + " LIMIT 0,1000"; + } + + @Override + public String createQuerySQL(String table, List fields) { + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + // 如果原始类型为时间 + if (f.getDeExtractType() == 1) { + if (f.getDeType() == 2 || f.getDeType() == 3) { + stringBuilder.append("UNIX_TIMESTAMP( ").append(f.getOriginName()).append(" )*1000 AS ").append(f.getDataeaseName()); + } else { + stringBuilder.append(" ").append(f.getOriginName()).append(" AS ").append(f.getDataeaseName()); + } + } else if (f.getDeExtractType() == 0) { + if (f.getDeType() == 2) { + stringBuilder.append("CAST( ").append(f.getOriginName()).append(" AS DECIMAL(20,0)) AS ").append(f.getDataeaseName()); + } else if (f.getDeType() == 3) { + stringBuilder.append("CAST( ").append(f.getOriginName()).append(" AS DECIMAL(20,2)) AS ").append(f.getDataeaseName()); + } else if (f.getDeType() == 1) { + stringBuilder.append("DATE_FORMAT( ").append(f.getOriginName()).append(" ,'%Y-%m-%d %H:%i:%S') AS _").append(f.getDataeaseName()); + } else { + stringBuilder.append(" ").append(f.getOriginName()).append(" AS ").append(f.getDataeaseName()); + } + } else { + if (f.getDeType() == 1) { + stringBuilder.append("FROM_UNIXTIME(CAST( ").append(f.getOriginName()).append(" AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') AS ").append(f.getDataeaseName()); + } else { + stringBuilder.append(" ").append(f.getOriginName()).append(" AS ").append(f.getDataeaseName()); + } + } + return stringBuilder.toString(); + }).toArray(String[]::new); + return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table); + } + + @Override + public String createQuerySQLAsTmp(String sql, List fields) { + return createQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields); + } + + @Override + public String createQuerySQLWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize) { + return createQuerySQL(table, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize; + } + + @Override + public String createQuerySQLAsTmpWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize) { + return createQuerySQLAsTmp(sql, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize; + } + + @Override + public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { + // 字段汇总 排序等 + String[] field = yAxis.stream().map(y -> { + StringBuilder f = new StringBuilder(); + if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) { + f.append(y.getSummary()).append("(").append(y.getOriginName()).append(")"); + } else { + if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { + f.append("CAST(") + .append(y.getSummary()).append("(") + .append("CAST( ").append(y.getOriginName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") + .append(") AS DECIMAL(20,2)").append(")"); + } else { + f.append(y.getSummary()).append("(") + .append("CAST( ").append(y.getOriginName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") + .append(")"); + } + } + f.append(" AS _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append(" "); + return f.toString(); + }).toArray(String[]::new); + String[] groupField = xAxis.stream().map(x -> { + StringBuilder stringBuilder = new StringBuilder(); + // 如果原始类型为时间 + if (x.getDeExtractType() == 1) { + if (x.getDeType() == 2 || x.getDeType() == 3) { + stringBuilder.append("UNIX_TIMESTAMP( ").append(x.getOriginName()).append(" )*1000 AS _").append(x.getOriginName()).append(" "); + } else if (x.getDeType() == 1) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + stringBuilder.append("DATE_FORMAT( ").append(x.getOriginName()).append(" ,'").append(format).append("') AS _").append(x.getOriginName()).append(" "); + } else { + stringBuilder.append(" ").append(x.getOriginName()).append(" AS _").append(x.getOriginName()).append(" "); + } + } else { + if (x.getDeType() == 1) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + if (x.getDeExtractType() == 0) { + stringBuilder.append("DATE_FORMAT( ").append(x.getOriginName()).append(" ,'").append(format).append("') AS _").append(x.getOriginName()).append(" "); + } else { + stringBuilder.append("DATE_FORMAT(").append("FROM_UNIXTIME(CAST( ").append(x.getOriginName()).append(" AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S')").append(",'").append(format).append("') AS _").append(x.getOriginName()).append(" "); + } + } else { + stringBuilder.append(" ").append(x.getOriginName()).append(" AS _").append(x.getOriginName()).append(" "); + } + } + return stringBuilder.toString(); + }).toArray(String[]::new); + String[] group = xAxis.stream().map(x -> " _" + x.getOriginName() + " ").toArray(String[]::new); + String[] xOrder = xAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) + .map(f -> " _" + f.getOriginName() + " " + f.getSort()).toArray(String[]::new); + String[] yOrder = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) + .map(f -> " _" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getOriginName(), "*") ? "" : f.getOriginName()) + " " + f.getSort()).toArray(String[]::new); + String[] order = Arrays.copyOf(xOrder, xOrder.length + yOrder.length); + System.arraycopy(yOrder, 0, order, xOrder.length, yOrder.length); + + String[] xFilter = xAxis.stream().filter(x -> CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) + .map(x -> { + String[] s = x.getFilter().stream().map(f -> { + StringBuilder filter = new StringBuilder(); + if (x.getDeType() == 1 && x.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(cast( ") + .append(x.getOriginName()) + .append(" AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND ").append(x.getOriginName()).append(" "); + } + filter.append(transMysqlFilterTerm(f.getTerm())); + if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + filter.append("%").append(f.getValue()).append("%"); + } else { + filter.append("'").append(f.getValue()).append("'"); + } + return filter.toString(); + }).toArray(String[]::new); + return StringUtils.join(s, " "); + }).toArray(String[]::new); + + String sql = MessageFormat.format("SELECT {0},{1} FROM {2} WHERE 1=1 {3} GROUP BY {4} ORDER BY null,{5}", + StringUtils.join(groupField, ","), + StringUtils.join(field, ","), + table, + (xFilter.length > 0 ? StringUtils.join(xFilter, " ") : "") + transCustomFilter(customFilter) + transExtFilter(extFilterRequestList),// origin field filter and panel field filter + StringUtils.join(group, ","), + StringUtils.join(order, ",")); + if (sql.endsWith(",")) { + sql = sql.substring(0, sql.length() - 1); + } + // 如果是对结果字段过滤，则再包裹一层sql + String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) + .map(y -> { + String[] s = y.getFilter().stream().map(f -> { + StringBuilder filter = new StringBuilder(); + // 原始类型不是时间，在de中被转成时间的字段做处理 + if (y.getDeType() == 1 && y.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(CAST( _") + .append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append(" ") + .append(" AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append(" "); + } + filter.append(transMysqlFilterTerm(f.getTerm())); + if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + filter.append("%").append(f.getValue()).append("%"); + } else { + filter.append("'").append(f.getValue()).append("'"); + } + return filter.toString(); + }).toArray(String[]::new); + return StringUtils.join(s, " "); + }).toArray(String[]::new); + if (resultFilter.length == 0) { + return sql; + } else { + String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}", + "(" + sql + ") AS tmp", + StringUtils.join(resultFilter, " "), + StringUtils.join(yOrder, ",")); + return filterSql; + } + } + + @Override + public String getSQLAsTmp(String sql, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { + return getSQL(" (" + sqlFix(sql) + ") AS tmp ", xAxis, yAxis, customFilter, extFilterRequestList); + } + + @Override + public String searchTable(String table) { + return "SELECT table_name FROM information_schema.TABLES WHERE table_name ='" + table + "'"; + } + + @Override + public String getSQLSummary(String table, List yAxis, List customFilter, List extFilterRequestList) { + // 字段汇总 排序等 + String[] field = yAxis.stream().map(y -> { + StringBuilder f = new StringBuilder(); + if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) { + f.append(y.getSummary()).append("(").append(y.getOriginName()).append(")"); + } else { + if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { + f.append("CAST(") + .append(y.getSummary()).append("(") + .append("CAST( ").append(y.getOriginName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") + .append(") AS DECIMAL(20,2)").append(")"); + } else { + f.append(y.getSummary()).append("(") + .append("CAST( ").append(y.getOriginName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") + .append(")"); + } + } + f.append(" AS _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append(" "); + return f.toString(); + }).toArray(String[]::new); + + String[] order = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) + .map(f -> " _" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getOriginName(), "*") ? "" : f.getOriginName()) + " " + f.getSort()).toArray(String[]::new); + + String sql = MessageFormat.format("SELECT {0} FROM {1} WHERE 1=1 {2} ORDER BY null,{3}", + StringUtils.join(field, ","), + table, + transCustomFilter(customFilter) + transExtFilter(extFilterRequestList),// origin field filter and panel field filter + StringUtils.join(order, ",")); + if (sql.endsWith(",")) { + sql = sql.substring(0, sql.length() - 1); + } + // 如果是对结果字段过滤，则再包裹一层sql + String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) + .map(y -> { + String[] s = y.getFilter().stream().map(f -> { + StringBuilder filter = new StringBuilder(); + // 原始类型不是时间，在de中被转成时间的字段做处理 + if (y.getDeType() == 1 && y.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(CAST( _") + .append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append(" ") + .append(" AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append(" "); + } + filter.append(transMysqlFilterTerm(f.getTerm())); + if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + filter.append("%").append(f.getValue()).append("%"); + } else { + filter.append("'").append(f.getValue()).append("'"); + } + return filter.toString(); + }).toArray(String[]::new); + return StringUtils.join(s, " "); + }).toArray(String[]::new); + if (resultFilter.length == 0) { + return sql; + } else { + String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}", + "(" + sql + ") AS tmp", + StringUtils.join(resultFilter, " "), + StringUtils.join(order, ",")); + return filterSql; + } + } + + @Override + public String getSQLSummaryAsTmp(String sql, List yAxis, List customFilter, List extFilterRequestList) { + return getSQLSummary(" (" + sqlFix(sql) + ") AS tmp ", yAxis, customFilter, extFilterRequestList); + } + + public String transMysqlFilterTerm(String term) { + switch (term) { + case "eq": + return " = "; + case "not_eq": + return " <> "; + case "lt": + return " < "; + case "le": + return " <= "; + case "gt": + return " > "; + case "ge": + return " >= "; + case "in": + return " IN "; + case "not in": + return " NOT IN "; + case "like": + return " LIKE "; + case "not like": + return " NOT LIKE "; + case "null": + return " IS NULL "; + case "not_null": + return " IS NOT NULL "; + case "between": + return " BETWEEN "; + default: + return ""; + } + } + + public String transCustomFilter(List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return ""; + } + StringBuilder filter = new StringBuilder(); + for (ChartCustomFilterDTO request : requestList) { + String value = request.getValue(); + DatasetTableField field = request.getField(); + if (ObjectUtils.isEmpty(field)) { + continue; + } + if (field.getDeType() == 1 && field.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(CAST( ") + .append(field.getOriginName()) + .append(" AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND ").append(field.getOriginName()).append(" "); + } + filter.append(" ") + .append(transMysqlFilterTerm(request.getTerm())) + .append(" "); + if (StringUtils.containsIgnoreCase(request.getTerm(), "null")) { + } else if (StringUtils.containsIgnoreCase(request.getTerm(), "in")) { + filter.append("('").append(StringUtils.join(value, "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(request.getTerm(), "like")) { + filter.append("'%").append(value).append("%'"); + } else { + filter.append("'").append(value).append("'"); + } + } + return filter.toString(); + } + + public String transExtFilter(List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return ""; + } + StringBuilder filter = new StringBuilder(); + for (ChartExtFilterRequest request : requestList) { + List value = request.getValue(); + if (CollectionUtils.isEmpty(value)) { + continue; + } + DatasetTableField field = request.getDatasetTableField(); + if (field.getDeType() == 1 && field.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(CAST( ") + .append(field.getOriginName()) + .append(" AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND ").append(field.getOriginName()).append(" "); + } + filter.append(" ") + .append(transMysqlFilterTerm(request.getOperator())) + .append(" "); + if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { + filter.append("('").append(StringUtils.join(value, "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { + filter.append("'%").append(value.get(0)).append("%'"); + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0)))); + String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1)))); + filter.append("'").append(startTime).append("' AND '").append(endTime).append("'"); + } else { + filter.append("'").append(value.get(0)).append("'"); + } + } + return filter.toString(); + } + + private String sqlFix(String sql) { + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + return sql; + } + + private String transDateFormat(String dateStyle, String datePattern) { + String split = "-"; + if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) { + split = "-"; + } else if (StringUtils.equalsIgnoreCase(datePattern, "date_split")) { + split = "/"; + } + + switch (dateStyle) { + case "y": + return "%Y"; + case "y_M": + return "%Y" + split + "%m"; + case "y_M_d": + return "%Y" + split + "%m" + split + "%d"; + case "H_m_s": + return "%H:%i:%S"; + case "y_M_d_H_m": + return "%Y" + split + "%m" + split + "%d" + " %H:%i"; + case "y_M_d_H_m_s": + return "%Y" + split + "%m" + split + "%d" + " %H:%i:%S"; + default: + return "%Y-%m-%d %H:%i:%S"; + } + } +} diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 4497013a31..addb66f675 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -293,6 +293,7 @@ public class DataSetTableService { String table = dataTableInfoDTO.getTable(); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize)); + System.out.println(datasourceRequest.getQuery()); try { data.addAll(datasourceProvider.getData(datasourceRequest)); } catch (Exception e) { diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 7faa2b5a73..35c827ef67 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -12,10 +12,7 @@ import io.dataease.commons.utils.CommonBeanFactory; import io.dataease.commons.utils.DorisTableUtils; import io.dataease.commons.utils.LogUtil; import io.dataease.datasource.constants.DatasourceTypes; -import io.dataease.datasource.dto.DorisConfigration; -import io.dataease.datasource.dto.MysqlConfigration; -import io.dataease.datasource.dto.SqlServerConfigration; -import io.dataease.datasource.dto.TableFiled; +import io.dataease.datasource.dto.*; import io.dataease.datasource.provider.DatasourceProvider; import io.dataease.datasource.provider.JdbcProvider; import io.dataease.datasource.provider.ProviderFactory; @@ -200,13 +197,13 @@ public class ExtractDataService { extractData(datasetTable, "all_scope"); replaceTable(DorisTableUtils.dorisName(datasetTableId)); saveSucessLog(datasetTableTaskLog); - deleteFile("all_scope", datasetTableId); +// deleteFile("all_scope", datasetTableId); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); }catch (Exception e){ saveErrorLog(datasetTableId, taskId, e); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); - deleteFile("all_scope", datasetTableId); +// deleteFile("all_scope", datasetTableId); }finally { if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString()); @@ -633,6 +630,29 @@ public class ExtractDataService { inputStep = inputStep(transMeta, selectSQL); udjcStep = udjc(datasetTableFields, false); break; + case oracle: + OracleConfigration oracleConfigration = new Gson().fromJson(datasource.getConfiguration(), OracleConfigration.class); + System.out.println(new Gson().toJson(oracleConfigration)); + System.out.println(oracleConfigration.getConnectionType().equalsIgnoreCase("serviceName")); + if(oracleConfigration.getConnectionType().equalsIgnoreCase("serviceName")){ + String database = "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = ORACLE_HOSTNAME)(PORT = ORACLE_PORT))(CONNECT_DATA = (SERVER = DEDICATED)(SERVICE_NAME = ORACLE_SERVICE_NAME )))".replace("ORACLE_HOSTNAME", oracleConfigration.getHost()).replace("ORACLE_PORT", oracleConfigration.getPort().toString()).replace("ORACLE_SERVICE_NAME", oracleConfigration.getDataBase()); + dataMeta = new DatabaseMeta("db", "ORACLE", "Native", "", database, "-1", oracleConfigration.getUsername(), oracleConfigration.getPassword()); + }else { + dataMeta = new DatabaseMeta("db", "ORACLE", "Native", oracleConfigration.getHost(), oracleConfigration.getDataBase(), oracleConfigration.getPort().toString(), oracleConfigration.getUsername(), oracleConfigration.getPassword()); + } + transMeta.addDatabase(dataMeta); + if (extractType.equalsIgnoreCase("all_scope")) { + if(datasetTable.getType().equalsIgnoreCase("sql")){ + selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(); + }else { + String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); + QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); + selectSQL = qp.createQuerySQL(tableName, datasetTableFields); + } + } + inputStep = inputStep(transMeta, selectSQL); + udjcStep = udjc(datasetTableFields, false); + break; case excel: String filePath = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData(); inputStep = excelInputStep(filePath, datasetTableFields); @@ -752,7 +772,7 @@ public class ExtractDataService { String needToChangeColumnType = ""; for (DatasetTableField datasetTableField : datasetTableFields) { if (datasetTableField.getDeExtractType() != null && datasetTableField.getDeExtractType() == 4) { - needToChangeColumnType = needToChangeColumnType + alterColumnTypeCode.replace("FILED", datasetTableField.getOriginName()); + needToChangeColumnType = needToChangeColumnType + alterColumnTypeCode.replace("FILED", datasetTableField.getDataeaseName()); } } @@ -762,7 +782,7 @@ public class ExtractDataService { fields.add(fieldInfo); userDefinedJavaClassMeta.setFieldInfo(fields); List definitions = new ArrayList(); - String tmp_code = code.replace("alterColumnTypeCode", needToChangeColumnType).replace("Column_Fields", String.join(",", datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList()))); + String tmp_code = code.replace("alterColumnTypeCode", needToChangeColumnType).replace("Column_Fields", String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()))); tmp_code = tmp_code.replace("handleWraps", handleWraps); if(isExcel){ tmp_code = tmp_code.replace("handleExcelIntColumn", handleExcelIntColumn); diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 3c24007e28..258eec2f17 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -860,6 +860,7 @@ export default { host: '主机名/IP地址', port: '端口', please_input_data_base: '请输入数据库名称', + please_select_oracle_type: '选择连接类型', please_input_user_name: '请输入用户名', please_input_password: '请输入密码', please_input_host: '请输入主机', @@ -871,7 +872,10 @@ export default { delete_warning: '确定要删除吗?', input_name: '请输入名称', input_limit_2_25: '2-25字符', - input_limit_0_50: '0-50字符' + input_limit_0_50: '0-50字符', + oracle_connection_type: '服务名/SID', + oracle_sid: 'SID', + oracle_service_name: '服务名', }, pblink: { key_pwd: '请输入密码打开链接', diff --git a/frontend/src/views/system/datasource/form.vue b/frontend/src/views/system/datasource/form.vue index 71b260931e..0d74dedcbf 100644 --- a/frontend/src/views/system/datasource/form.vue +++ b/frontend/src/views/system/datasource/form.vue @@ -31,6 +31,12 @@ + + + {{ $t('datasource.oracle_sid') }} + {{ $t('datasource.oracle_service_name') }} + + @@ -40,16 +46,6 @@ - -
{{ $t('commons.validate') }} @@ -83,25 +79,19 @@ export default { { min: 2, max: 25, message: this.$t('datasource.input_limit_2_25', [2, 25]), trigger: 'blur' }], desc: [{ min: 0, max: 50, message: this.$t('datasource.input_limit_0_50'), trigger: 'blur' }], type: [{ required: true, message: this.$t('datasource.please_choose_type'), trigger: 'change' }], - 'configuration.dataBase': [{ required: true, message: this.$t('datasource.please_input_data_base'), trigger: 'blur' }], + 'configuration.connectionType': [{ required: true, message: this.$t('datasource.please_select_oracle_type'), trigger: 'blur' }], 'configuration.username': [{ required: true, message: this.$t('datasource.please_input_user_name'), trigger: 'blur' }], 'configuration.password': [{ required: true, message: this.$t('datasource.please_input_password'), trigger: 'change' }], 'configuration.host': [{ required: true, message: this.$t('datasource.please_input_host'), trigger: 'change' }], 'configuration.port': [{ required: true, message: this.$t('datasource.please_input_port'), trigger: 'change' }] }, - allTypes: [{ name: 'mysql', label: 'MySQL', type: 'jdbc' }], + allTypes: [{ name: 'mysql', label: 'MySQL', type: 'jdbc' }, { name: 'oracle', label: 'Oracle', type: 'jdbc' }], canEdit: false } }, created() { - // if (this.$router.currentRoute.params && this.$router.currentRoute.params.id) { - // const row = this.$router.currentRoute.params - // this.edit(row) - // } else { - // this.create() - // } if (this.params && this.params.id) { const row = this.params this.edit(row) @@ -113,12 +103,6 @@ export default { } }, mounted() { - // if (this.params && this.params.type) { - // this.form.type = this.params.type - // this.$nextTick(() => { - // this.changeType() - // }) - // } }, methods: { setType() {