center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge pull request #8208 from dataease/pr@dev-v2@refactor_calcite_performance

Browse Source 
refactor: 优化calcite自定义函数性能
This commit is contained in:
Junjun 2024-02-28 13:01:10 +08:00 committed by GitHub
commit bb95102c97
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 15 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java
View File

@ -25,11 +25,9 @@ import io.dataease.utils.LogUtil;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import jakarta.annotation.Resource; import jakarta.annotation.Resource;
import org.apache.calcite.adapter.jdbc.JdbcSchema; import org.apache.calcite.adapter.jdbc.JdbcSchema;
import org.apache.calcite.func.scalar.ScalarFunctions;
import org.apache.calcite.jdbc.CalciteConnection; import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.schema.impl.ScalarFunctionImpl;
import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@ -37,7 +35,6 @@ import org.springframework.util.CollectionUtils;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.lang.reflect.Method;
import java.net.URL; import java.net.URL;
import java.sql.*; import java.sql.*;
import java.util.*; import java.util.*;
@ -206,20 +203,9 @@ public class CalciteProvider {
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(dsMap); datasourceRequest.setDsList(dsMap);
SchemaPlus rootSchema = buildSchema(datasourceRequest, calciteConnection); SchemaPlus rootSchema = buildSchema(datasourceRequest, calciteConnection);
addCustomFunctions(rootSchema);
return connection; return connection;
} }
private void addCustomFunctions(SchemaPlus rootSchema) {
// scalar functions
Class<?> clazz = ScalarFunctions.class;
Method[] methods = clazz.getMethods();
for (Method method : methods) {
rootSchema.add(method.getName().toUpperCase(), ScalarFunctionImpl.create(
ScalarFunctions.class, method.getName()));
}
}
private void registerDriver() { private void registerDriver() {
for (String driverClass : getDriver()) { for (String driverClass : getDriver()) {
try { try {
@ -691,7 +677,6 @@ public class CalciteProvider {
try { try {
CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class); CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
SchemaPlus rootSchema = buildSchema(datasourceRequest, calciteConnection); SchemaPlus rootSchema = buildSchema(datasourceRequest, calciteConnection);
addCustomFunctions(rootSchema);
} catch (Exception e) { } catch (Exception e) {
DEException.throwException(e.getMessage()); DEException.throwException(e.getMessage());
} }

10
core/core-backend/src/main/java/io/dataease/engine/trans/CustomWhere2Str.java
View File

@ -6,11 +6,12 @@ import io.dataease.api.dataset.union.model.SQLMeta;
import io.dataease.api.dataset.union.model.SQLObj; import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.utils.DateUtils;
import io.dataease.engine.utils.Utils; import io.dataease.engine.utils.Utils;
import org.apache.calcite.func.scalar.ScalarFunctions;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -55,7 +56,7 @@ public class CustomWhere2Str {
} }
if (field.getDeExtractType() == 1) { if (field.getDeExtractType() == 1) {
// 此处获取标准格式的日期 // 此处获取标准格式的日期
String f = ScalarFunctions.get_date_format(originName); String f = DateUtils.get_date_format(originName);
whereName = String.format(SQLConstants.DATE_FORMAT, originName, f); whereName = String.format(SQLConstants.DATE_FORMAT, originName, f);
} }
} else if (field.getDeType() == 2 || field.getDeType() == 3) { } else if (field.getDeType() == 2 || field.getDeType() == 3) {
@ -90,11 +91,11 @@ public class CustomWhere2Str {
if (field.getDeType() == 1) { if (field.getDeType() == 1) {
// 规定几种日期格式，一一匹配，匹配到就是该格式 // 规定几种日期格式，一一匹配，匹配到就是该格式
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
String f = ScalarFunctions.get_date_format(filterItemDTO.getValue()); String f = DateUtils.get_date_format(filterItemDTO.getValue());
String n = String.format(SQLConstants.DE_CAST_DATE_FORMAT, whereName, StringUtils.isNotEmpty(field.getDateFormat()) ? field.getDateFormat() : SQLConstants.DEFAULT_DATE_FORMAT, f); String n = String.format(SQLConstants.DE_CAST_DATE_FORMAT, whereName, StringUtils.isNotEmpty(field.getDateFormat()) ? field.getDateFormat() : SQLConstants.DEFAULT_DATE_FORMAT, f);
whereNameReal = String.format(SQLConstants.UNIX_TIMESTAMP, n); whereNameReal = String.format(SQLConstants.UNIX_TIMESTAMP, n);
} else { } else {
String f = ScalarFunctions.get_date_format(filterItemDTO.getValue()); String f = DateUtils.get_date_format(filterItemDTO.getValue());
String n = String.format(SQLConstants.DE_DATE_FORMAT, whereName, f); String n = String.format(SQLConstants.DE_DATE_FORMAT, whereName, f);
whereNameReal = String.format(SQLConstants.UNIX_TIMESTAMP, n); whereNameReal = String.format(SQLConstants.UNIX_TIMESTAMP, n);
} }
@ -136,4 +137,5 @@ public class CustomWhere2Str {
} }
} }
} }

4
core/core-backend/src/main/java/io/dataease/engine/trans/ExtWhere2Str.java
View File

@ -5,8 +5,8 @@ import io.dataease.api.dataset.union.model.SQLMeta;
import io.dataease.api.dataset.union.model.SQLObj; import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.utils.DateUtils;
import io.dataease.engine.utils.Utils; import io.dataease.engine.utils.Utils;
import org.apache.calcite.func.scalar.ScalarFunctions;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -57,7 +57,7 @@ public class ExtWhere2Str {
if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) { if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) {
date_format = "yyyy-MM-dd HH:mm:ss"; date_format = "yyyy-MM-dd HH:mm:ss";
} else { } else {
date_format = ScalarFunctions.get_date_format(value.get(0)); date_format = DateUtils.get_date_format(value.get(0));
} }
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
// 此处获取标准格式的日期 // 此处获取标准格式的日期

12
core/core-backend/src/main/java/io/dataease/engine/trans/WhereTree2Str.java
View File

@ -8,8 +8,8 @@ import io.dataease.api.permissions.dataset.dto.DatasetRowPermissionsTreeObj;
import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.ExtFieldConstant; import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.utils.DateUtils;
import io.dataease.engine.utils.Utils; import io.dataease.engine.utils.Utils;
import org.apache.calcite.func.scalar.ScalarFunctions;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -98,7 +98,7 @@ public class WhereTree2Str {
whereName = String.format(SQLConstants.FROM_UNIXTIME, cast, SQLConstants.DEFAULT_DATE_FORMAT); whereName = String.format(SQLConstants.FROM_UNIXTIME, cast, SQLConstants.DEFAULT_DATE_FORMAT);
} }
if (field.getDeExtractType() == 1) { if (field.getDeExtractType() == 1) {
String f = ScalarFunctions.get_date_format(originName); String f = DateUtils.get_date_format(originName);
whereName = String.format(SQLConstants.DATE_FORMAT, originName, f); whereName = String.format(SQLConstants.DATE_FORMAT, originName, f);
} }
} else if (field.getDeType() == 2 || field.getDeType() == 3) { } else if (field.getDeType() == 2 || field.getDeType() == 3) {
@ -130,12 +130,12 @@ public class WhereTree2Str {
if (field.getDeType() == 1) { if (field.getDeType() == 1) {
// 规定几种日期格式，一一匹配，匹配到就是该格式 // 规定几种日期格式，一一匹配，匹配到就是该格式
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
String f = ScalarFunctions.get_date_format(item.getValue()); String f = DateUtils.get_date_format(item.getValue());
whereName = String.format(SQLConstants.DE_CAST_DATE_FORMAT, whereName, StringUtils.isNotEmpty(field.getDateFormat()) ? field.getDateFormat() : SQLConstants.DEFAULT_DATE_FORMAT, f); whereName = String.format(SQLConstants.DE_STR_TO_DATE, whereName, f);
whereName = String.format(SQLConstants.UNIX_TIMESTAMP, whereName); whereName = String.format(SQLConstants.UNIX_TIMESTAMP, whereName);
} else { } else {
String f = ScalarFunctions.get_date_format(item.getValue()); String f = DateUtils.get_date_format(item.getValue());
whereName = String.format(SQLConstants.DE_DATE_FORMAT, whereName, f); whereName = String.format(SQLConstants.DE_STR_TO_DATE, whereName, f);
whereName = String.format(SQLConstants.UNIX_TIMESTAMP, whereName); whereName = String.format(SQLConstants.UNIX_TIMESTAMP, whereName);
} }
} }

2
pom.xml
View File

@ -25,7 +25,7 @@
<mybatis-plus.version>3.5.3.1</mybatis-plus.version> <mybatis-plus.version>3.5.3.1</mybatis-plus.version>
<h2.version>2.2.220</h2.version> <h2.version>2.2.220</h2.version>
<knife4j.version>4.4.0</knife4j.version> <knife4j.version>4.4.0</knife4j.version>
<calcite-core.version>1.35.4</calcite-core.version> <calcite-core.version>1.35.5</calcite-core.version>
<commons-dbcp2.version>2.6.0</commons-dbcp2.version> <commons-dbcp2.version>2.6.0</commons-dbcp2.version>
<antlr.version>3.5.2</antlr.version> <antlr.version>3.5.2</antlr.version>
<java-jwt.version>3.12.1</java-jwt.version> <java-jwt.version>3.12.1</java-jwt.version>