center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge pull request #128 from dataease/pr@dev@oracle

Browse Source 
feat: oracle 增量更新
This commit is contained in:
taojinlong 2021-07-01 15:56:08 +08:00 committed by GitHub
commit e6089deaee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 47 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/io/dataease/provider/QueryProvider.java
View File

@ -41,4 +41,6 @@ public abstract class QueryProvider {
public abstract String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList);
public abstract String getSQLSummaryAsTmp(String sql, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList);
public abstract String wrapSql(String sql);
}

10
backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java
View File

@ -338,6 +338,16 @@ public class DorisQueryProvider extends QueryProvider {
return getSQLSummary(" (" + sql + ") AS tmp ", yAxis, customFilter, extFilterRequestList);
}
@Override
public String wrapSql(String sql) {
sql = sql.trim();
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0";
return tmpSql;
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":

10
backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java
View File

@ -346,6 +346,16 @@ public class MysqlQueryProvider extends QueryProvider {
return getSQLSummary(" (" + sqlFix(sql) + ") AS tmp ", yAxis, customFilter, extFilterRequestList);
}
@Override
public String wrapSql(String sql) {
sql = sql.trim();
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0";
return tmpSql;
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":

10
backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java
View File

@ -490,6 +490,16 @@ public class OracleQueryProvider extends QueryProvider {
return sql;
}
@Override
public String wrapSql(String sql) {
sql = sql.trim();
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
String tmpSql = "SELECT * FROM (" + sql + ") tmp " + " where rownum <= 0";
return tmpSql;
}
private String transDateFormat(String dateStyle, String datePattern) {
String split = "-";
if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) {

10
backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java
View File

@ -260,6 +260,16 @@ public class SqlserverQueryProvider extends QueryProvider {
return null;
}
@Override
public String wrapSql(String sql) {
sql = sql.trim();
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0";
return tmpSql;
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":

5
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -721,13 +721,14 @@ public class DataSetTableService {
});
List<String> originNameFileds = datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList());
Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString())
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
datasourceRequest.setQuery(extractDataService.sqlFix(sql));
datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> sqlFileds = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed -> {
sqlFileds.add(filed);
@ -740,7 +741,7 @@ public class DataSetTableService {
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除
String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString())
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
datasourceRequest.setQuery(extractDataService.sqlFix(sql));
datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> sqlFileds = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed -> {
sqlFileds.add(filed);

13
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -580,10 +580,11 @@ public class ExtractDataService {
}
private String fetchSqlField(String sql, Datasource ds) throws Exception {
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
datasourceRequest.setQuery(sqlFix(sql));
datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> dorisFileds = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{
dorisFileds.add(DorisTableUtils.columnName(filed));
@ -591,16 +592,6 @@ public class ExtractDataService {
return String.join(",", dorisFileds);
}
public String sqlFix(String sql) {
sql = sql.trim();
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0";
return tmpSql;
}
private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String selectSQL) throws Exception {
TransMeta transMeta = new TransMeta();
String outFile = null;