center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge pull request #130 from dataease/pr@dev@oracle

Browse Source 
feat: 优化 oracle 增量更新
This commit is contained in:
taojinlong 2021-07-01 18:21:05 +08:00 committed by GitHub
commit bd2bfa0a24
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 110 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/main/java/io/dataease/provider/QueryProvider.java
View File

@ -43,4 +43,8 @@ public abstract class QueryProvider {
public abstract String getSQLSummaryAsTmp(String sql, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList);
public abstract String wrapSql(String sql);
public abstract String createRawQuerySQL(String table, List<DatasetTableField> fields);
public abstract String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields);
}

22
backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java
View File

@ -348,6 +348,28 @@ public class DorisQueryProvider extends QueryProvider {
return tmpSql;
}
@Override
public String createRawQuerySQL(String table, List<DatasetTableField> fields){
String[] array = fields.stream().map(f -> {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName());
return stringBuilder.toString();
}).toArray(String[]::new);
return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
}
@Override
public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields);
}
private String sqlFix(String sql) {
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
return sql;
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":

17
backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java
View File

@ -102,8 +102,6 @@ public class MysqlQueryProvider extends QueryProvider {
}
return stringBuilder.toString();
}).toArray(String[]::new);
// return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getOriginName() : "null"), StringUtils.join(array, ","), table);
return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
}
@ -356,6 +354,21 @@ public class MysqlQueryProvider extends QueryProvider {
return tmpSql;
}
@Override
public String createRawQuerySQL(String table, List<DatasetTableField> fields){
String[] array = fields.stream().map(f -> {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName());
return stringBuilder.toString();
}).toArray(String[]::new);
return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
}
@Override
public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields);
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":

35
backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java
View File

@ -380,6 +380,31 @@ public class OracleQueryProvider extends QueryProvider {
return getSQLSummary(" (" + sqlFix(sql) + ") tmp ", yAxis, customFilter, extFilterRequestList);
}
@Override
public String wrapSql(String sql) {
sql = sql.trim();
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
String tmpSql = "SELECT * FROM (" + sql + ") tmp " + " where rownum <= 0";
return tmpSql;
}
@Override
public String createRawQuerySQL(String table, List<DatasetTableField> fields){
String[] array = fields.stream().map(f -> {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(" ").append(f.getOriginName()).append(" AS ").append(f.getDataeaseName());
return stringBuilder.toString();
}).toArray(String[]::new);
return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
}
@Override
public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
return createRawQuerySQL(" (" + sqlFix(sql) + ") tmp ", fields);
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":
@ -490,16 +515,6 @@ public class OracleQueryProvider extends QueryProvider {
return sql;
}
@Override
public String wrapSql(String sql) {
sql = sql.trim();
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
String tmpSql = "SELECT * FROM (" + sql + ") tmp " + " where rownum <= 0";
return tmpSql;
}
private String transDateFormat(String dateStyle, String datePattern) {
String split = "-";
if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) {

15
backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java
View File

@ -270,6 +270,21 @@ public class SqlserverQueryProvider extends QueryProvider {
return tmpSql;
}
@Override
public String createRawQuerySQL(String table, List<DatasetTableField> fields){
String[] array = fields.stream().map(f -> {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName());
return stringBuilder.toString();
}).toArray(String[]::new);
return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
}
@Override
public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields);
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":

62
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -206,13 +206,13 @@ public class ExtractDataService {
extractData(datasetTable, "all_scope");
replaceTable(DorisTableUtils.dorisName(datasetTableId));
saveSucessLog(datasetTableTaskLog);
deleteFile("all_scope", datasetTableId);
// deleteFile("all_scope", datasetTableId);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
}catch (Exception e){
saveErrorLog(datasetTableId, taskId, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
deleteFile("all_scope", datasetTableId);
// deleteFile("all_scope", datasetTableId);
}finally {
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString());
@ -267,15 +267,15 @@ public class ExtractDataService {
extractData(datasetTable, "incremental_delete");
}
saveSucessLog(datasetTableTaskLog);
deleteFile("incremental_add", datasetTableId);
deleteFile("incremental_delete", datasetTableId);
// deleteFile("incremental_add", datasetTableId);
// deleteFile("incremental_delete", datasetTableId);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
}
}catch (Exception e){
saveErrorLog(datasetTableId, taskId, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
deleteFile("incremental_add", datasetTableId);
deleteFile("incremental_delete", datasetTableId);
// deleteFile("incremental_add", datasetTableId);
// deleteFile("incremental_delete", datasetTableId);
}finally {
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString());
@ -610,15 +610,7 @@ public class ExtractDataService {
dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfigration.getHost(), mysqlConfigration.getDataBase(), mysqlConfigration.getPort().toString(), mysqlConfigration.getUsername(), mysqlConfigration.getPassword());
dataMeta.addExtraOption("MYSQL","characterEncoding", "UTF-8");
transMeta.addDatabase(dataMeta);
if (extractType.equalsIgnoreCase("all_scope")) {
if(datasetTable.getType().equalsIgnoreCase("sql")){
selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql();
}else {
String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
selectSQL = qp.createQuerySQL(tableName, datasetTableFields);
}
}
selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
inputStep = inputStep(transMeta, selectSQL);
udjcStep = udjc(datasetTableFields, false);
break;
@ -626,15 +618,7 @@ public class ExtractDataService {
SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasource.getConfiguration(), SqlServerConfigration.class);
dataMeta = new DatabaseMeta("db", "MSSQLNATIVE", "Native", sqlServerConfigration.getHost(), sqlServerConfigration.getDataBase(), sqlServerConfigration.getPort().toString(), sqlServerConfigration.getUsername(), sqlServerConfigration.getPassword());
transMeta.addDatabase(dataMeta);
if (extractType.equalsIgnoreCase("all_scope")) {
if(datasetTable.getType().equalsIgnoreCase("sql")){
selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql();
}else {
String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
selectSQL = qp.createQuerySQL(tableName, datasetTableFields);
}
}
selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
inputStep = inputStep(transMeta, selectSQL);
udjcStep = udjc(datasetTableFields, false);
break;
@ -647,15 +631,8 @@ public class ExtractDataService {
dataMeta = new DatabaseMeta("db", "ORACLE", "Native", oracleConfigration.getHost(), oracleConfigration.getDataBase(), oracleConfigration.getPort().toString(), oracleConfigration.getUsername(), oracleConfigration.getPassword());
}
transMeta.addDatabase(dataMeta);
if (extractType.equalsIgnoreCase("all_scope")) {
if(datasetTable.getType().equalsIgnoreCase("sql")){
selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql();
}else {
String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
selectSQL = qp.createQuerySQL(tableName, datasetTableFields);
}
}
selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
inputStep = inputStep(transMeta, selectSQL);
udjcStep = udjc(datasetTableFields, false);
break;
@ -702,6 +679,25 @@ public class ExtractDataService {
FileUtils.writeStringToFile(file, transXml, "UTF-8");
}
private String getSelectSQL(String extractType, DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String selectSQL) {
if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("db")) {
String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
selectSQL = qp.createRawQuerySQL(tableName, datasetTableFields);
}
if(extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("sql")){
selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql();
QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields);
}
if(!extractType.equalsIgnoreCase("all_scope")){
QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields);
}
return selectSQL;
}
private StepMeta inputStep(TransMeta transMeta, String selectSQL) {
TableInputMeta tableInput = new TableInputMeta();
DatabaseMeta database = transMeta.findDatabase("db");