diff --git a/backend/pom.xml b/backend/pom.xml
index c814656394..f22eacfbe2 100644
--- a/backend/pom.xml
+++ b/backend/pom.xml
@@ -303,17 +303,6 @@
reflections8
0.11.7
-
-
org.springframework.boot
@@ -325,6 +314,40 @@
ehcache
2.9.1
+
+
+ org.apache.hbase
+ hbase
+ 2.4.1
+ pom
+
+
+ org.apache.hbase
+ hbase-client
+ 2.4.1
+
+
+ org.apache.hbase
+ hbase-common
+ 2.4.1
+
+
+ org.apache.hbase
+ hbase-server
+ 2.4.1
+
+
+ org.apache.hbase
+ hbase-mapreduce
+ 2.4.1
+
+
+ org.testng
+ testng
+ 6.8
+ test
+
+
diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java b/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java
index 6e7288c65e..9cd293962b 100644
--- a/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java
+++ b/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java
@@ -1,9 +1,12 @@
package io.dataease.base.domain;
import java.io.Serializable;
+
+import lombok.Builder;
import lombok.Data;
@Data
+@Builder
public class DatasetTableField implements Serializable {
private String id;
@@ -24,4 +27,4 @@ public class DatasetTableField implements Serializable {
private Integer deType;
private static final long serialVersionUID = 1L;
-}
\ No newline at end of file
+}
diff --git a/backend/src/main/java/io/dataease/commons/constants/DatasetMode.java b/backend/src/main/java/io/dataease/commons/constants/DatasetMode.java
new file mode 100644
index 0000000000..b47fe5c05f
--- /dev/null
+++ b/backend/src/main/java/io/dataease/commons/constants/DatasetMode.java
@@ -0,0 +1,6 @@
+package io.dataease.commons.constants;
+
+public class DatasetMode {
+ public static final String EXTRACT = "1";
+ public static final String DIRECT = "0";
+}
diff --git a/backend/src/main/java/io/dataease/commons/constants/JobStatus.java b/backend/src/main/java/io/dataease/commons/constants/JobStatus.java
new file mode 100644
index 0000000000..59d7672641
--- /dev/null
+++ b/backend/src/main/java/io/dataease/commons/constants/JobStatus.java
@@ -0,0 +1,5 @@
+package io.dataease.commons.constants;
+
+public enum JobStatus {
+ Prepare, Underway, Completed, Error
+}
diff --git a/backend/src/main/java/io/dataease/commons/constants/TestPlanStatus.java b/backend/src/main/java/io/dataease/commons/constants/TestPlanStatus.java
deleted file mode 100644
index 24aefa1ce1..0000000000
--- a/backend/src/main/java/io/dataease/commons/constants/TestPlanStatus.java
+++ /dev/null
@@ -1,5 +0,0 @@
-package io.dataease.commons.constants;
-
-public enum TestPlanStatus {
- Prepare, Underway, Completed
-}
diff --git a/backend/src/main/java/io/dataease/config/HbaseConfig.java b/backend/src/main/java/io/dataease/config/HbaseConfig.java
new file mode 100644
index 0000000000..f7b8bfaece
--- /dev/null
+++ b/backend/src/main/java/io/dataease/config/HbaseConfig.java
@@ -0,0 +1,30 @@
+package io.dataease.config;
+
+import com.fit2cloud.autoconfigure.QuartzAutoConfiguration;
+import org.springframework.boot.autoconfigure.AutoConfigureBefore;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.DependsOn;
+import org.springframework.core.env.Environment;
+
+import javax.annotation.Resource;
+
+@Configuration
+@AutoConfigureBefore(QuartzAutoConfiguration.class)
+public class HbaseConfig {
+
+ @Resource
+ private Environment env; // 保存了配置文件的信息
+
+
+ @Bean
+ @ConditionalOnMissingBean
+ public org.apache.hadoop.conf.Configuration configuration(){
+ org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
+ configuration.set("hbase.zookeeper.quorum", env.getProperty("hbase.zookeeper.quorum"));
+ configuration.set("hbase.zookeeper.property.clientPort", env.getProperty("hbase.zookeeper.property.clientPort"));
+ configuration.set("hbase.client.retries.number", env.getProperty("hbase.client.retries.number", "1"));
+ return configuration;
+ }
+}
diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableFieldController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableFieldController.java
index 691948db33..d4d135ccfd 100644
--- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableFieldController.java
+++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableFieldController.java
@@ -19,7 +19,7 @@ public class DataSetTableFieldController {
@PostMapping("list/{tableId}")
public List list(@PathVariable String tableId) {
- DatasetTableField datasetTableField = new DatasetTableField();
+ DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(tableId);
return dataSetTableFieldsService.list(datasetTableField);
}
diff --git a/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java b/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java
index 42c0a1a20a..d1dc06956d 100644
--- a/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java
+++ b/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java
@@ -23,4 +23,8 @@ public abstract class DatasourceProvider {
getData(datasourceRequest);
}
+ abstract public Long count(DatasourceRequest datasourceRequest)throws Exception;
+
+ abstract public List getPageData(DatasourceRequest datasourceRequest) throws Exception;
+
}
diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java
index d3c94c8230..4b53abfb49 100644
--- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java
+++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java
@@ -9,6 +9,7 @@ import io.dataease.datasource.request.DatasourceRequest;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.sql.*;
+import java.text.MessageFormat;
import java.util.*;
@Service("jdbc")
@@ -23,23 +24,7 @@ public class JdbcProvider extends DatasourceProvider{
Statement stat = connection.createStatement();
ResultSet rs = stat.executeQuery(datasourceRequest.getQuery())
) {
- ResultSetMetaData metaData = rs.getMetaData();
- int columnCount = metaData.getColumnCount();
- while (rs.next()) {
- String[] row = new String[columnCount];
- for (int j = 0; j < columnCount; j++) {
- int columType = metaData.getColumnType(j + 1);
- switch (columType) {
- case java.sql.Types.DATE:
- row[j] = rs.getDate(j + 1).toString();
- break;
- default:
- row[j] = rs.getString(j + 1);
- break;
- }
- }
- list.add(row);
- }
+ list = fetchResult(rs);
} catch (SQLException e){
throw new Exception("ERROR:" + e.getMessage(), e);
}catch (Exception e) {
@@ -48,6 +33,46 @@ public class JdbcProvider extends DatasourceProvider{
return list;
}
+ @Override
+ public List getPageData(DatasourceRequest datasourceRequest) throws Exception {
+ List list = new LinkedList<>();
+ try (
+ Connection connection = getConnection(datasourceRequest);
+ Statement stat = connection.createStatement();
+ ResultSet rs = stat.executeQuery(datasourceRequest.getQuery() + MessageFormat.format(" LIMIT {0}, {1}", (datasourceRequest.getStartPage() -1)*datasourceRequest.getPageSize(), datasourceRequest.getPageSize()))
+ ) {
+ list = fetchResult(rs);
+ } catch (SQLException e){
+ throw new Exception("ERROR:" + e.getMessage(), e);
+ }catch (Exception e) {
+ throw new Exception("ERROR:" + e.getMessage(), e);
+ }
+ return list;
+ }
+
+
+ private List fetchResult( ResultSet rs) throws Exception{
+ List list = new LinkedList<>();
+ ResultSetMetaData metaData = rs.getMetaData();
+ int columnCount = metaData.getColumnCount();
+ while (rs.next()) {
+ String[] row = new String[columnCount];
+ for (int j = 0; j < columnCount; j++) {
+ int columType = metaData.getColumnType(j + 1);
+ switch (columType) {
+ case java.sql.Types.DATE:
+ row[j] = rs.getDate(j + 1).toString();
+ break;
+ default:
+ row[j] = rs.getString(j + 1);
+ break;
+ }
+ }
+ list.add(row);
+ }
+ return list;
+ }
+
@Override
public List getTables(DatasourceRequest datasourceRequest) throws Exception {
List tables = new ArrayList<>();
@@ -106,6 +131,19 @@ public class JdbcProvider extends DatasourceProvider{
}
}
+
+ public Long count(DatasourceRequest datasourceRequest)throws Exception{
+ try (Connection con = getConnection(datasourceRequest); Statement ps = con.createStatement()) {
+ ResultSet resultSet = ps.executeQuery(datasourceRequest.getQuery());
+ while (resultSet.next()) {
+ return resultSet.getLong(1);
+ }
+ } catch (Exception e) {
+ throw new Exception("ERROR: " + e.getMessage(), e);
+ }
+ return 0L;
+ }
+
private Connection getConnection(DatasourceRequest datasourceRequest) throws Exception {
String username = null;
String password = null;
diff --git a/backend/src/main/java/io/dataease/datasource/request/DatasourceRequest.java b/backend/src/main/java/io/dataease/datasource/request/DatasourceRequest.java
index 60d7b4f1b0..9521b50af5 100644
--- a/backend/src/main/java/io/dataease/datasource/request/DatasourceRequest.java
+++ b/backend/src/main/java/io/dataease/datasource/request/DatasourceRequest.java
@@ -11,5 +11,8 @@ public class DatasourceRequest {
protected String query;
protected String table;
protected Datasource datasource;
+ private Long pageSize;
+ private Long startPage;
+
}
diff --git a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java
index 6e2a336853..3c409114d8 100644
--- a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java
+++ b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java
@@ -1,22 +1,25 @@
package io.dataease.job.sechedule;
-import org.quartz.Job;
-import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
+import io.dataease.commons.utils.LogUtil;
+import org.quartz.*;
public abstract class DeScheduleJob implements Job {
+ protected String datasetTableId;
+ protected String expression;
+ protected String taskId;
+
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
+ JobKey jobKey = context.getTrigger().getJobKey();
+ JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
+ this.datasetTableId = jobDataMap.getString("datasetTableId");
+ this.expression = jobDataMap.getString("expression");
+ this.taskId = jobDataMap.getString("taskId");
-// JobKey jobKey = context.getTrigger().getJobKey();
-// JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
-// this.resourceId = jobDataMap.getString("resourceId");
-// this.userId = jobDataMap.getString("userId");
-// this.expression = jobDataMap.getString("expression");
-//
-// LogUtil.info(jobKey.getGroup() + " Running: " + resourceId);
-// LogUtil.info("CronExpression: " + expression);
+ LogUtil.info(jobKey.getGroup() + " Running: " + datasetTableId);
+ LogUtil.info(jobKey.getName() + " Running: " + datasetTableId);
+ LogUtil.info("CronExpression: " + expression);
businessExecute(context);
}
diff --git a/backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java b/backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java
new file mode 100644
index 0000000000..2704f33a02
--- /dev/null
+++ b/backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java
@@ -0,0 +1,22 @@
+package io.dataease.job.sechedule;
+
+import io.dataease.commons.utils.CommonBeanFactory;
+import io.dataease.service.dataset.ExtractDataService;
+import org.quartz.JobExecutionContext;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ExtractDataJob extends DeScheduleJob{
+ private ExtractDataService extractDataService;
+
+ public ExtractDataJob() {
+ extractDataService = (ExtractDataService) CommonBeanFactory.getBean(ExtractDataService.class);
+ }
+
+
+ @Override
+ void businessExecute(JobExecutionContext context) {
+ extractDataService.extractData(datasetTableId, taskId);
+ }
+
+}
diff --git a/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java b/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java
index 045ba851fb..44bdf3aaad 100644
--- a/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java
+++ b/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java
@@ -369,11 +369,11 @@ public class ScheduleManager {
addOrUpdateCronJob(jobKey, triggerKey, jobClass, cron, startTime, endTime, null);
}
- public JobDataMap getDefaultJobDataMap(String resourceId, String expression, String userId) {
+ public JobDataMap getDefaultJobDataMap(String resourceId, String expression, String taskId) {
JobDataMap jobDataMap = new JobDataMap();
- jobDataMap.put("resourceId", resourceId);
+ jobDataMap.put("datasetTableId", resourceId);
+ jobDataMap.put("taskId", taskId);
jobDataMap.put("expression", expression);
- jobDataMap.put("userId", userId);
return jobDataMap;
}
diff --git a/backend/src/main/java/io/dataease/listener/AppStartListener.java b/backend/src/main/java/io/dataease/listener/AppStartListener.java
index 11dfdfac91..f44b517d36 100644
--- a/backend/src/main/java/io/dataease/listener/AppStartListener.java
+++ b/backend/src/main/java/io/dataease/listener/AppStartListener.java
@@ -20,13 +20,7 @@ public class AppStartListener implements ApplicationListener list = dataSetTableTaskService.list(new DatasetTableTask());
for (DatasetTableTask task : list) {
diff --git a/backend/src/main/java/io/dataease/service/ScheduleService.java b/backend/src/main/java/io/dataease/service/ScheduleService.java
index 79739fa41e..5885687dcb 100644
--- a/backend/src/main/java/io/dataease/service/ScheduleService.java
+++ b/backend/src/main/java/io/dataease/service/ScheduleService.java
@@ -1,8 +1,8 @@
package io.dataease.service;
import io.dataease.base.domain.DatasetTableTask;
+import io.dataease.job.sechedule.ExtractDataJob;
import io.dataease.job.sechedule.ScheduleManager;
-import io.dataease.job.sechedule.TestJob;
import org.apache.commons.lang3.StringUtils;
import org.quartz.JobKey;
import org.quartz.TriggerKey;
@@ -24,8 +24,8 @@ public class ScheduleService {
if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), "0")) {
scheduleManager.addOrUpdateSingleJob(new JobKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
new TriggerKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
- TestJob.class,//TODO
- new Date(datasetTableTask.getStartTime()));
+ ExtractDataJob.class,
+ new Date(datasetTableTask.getStartTime()), scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId()));
} else if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), "1")) {
Date endTime;
if (datasetTableTask.getEndTime() == null || datasetTableTask.getEndTime() == 0) {
@@ -36,8 +36,9 @@ public class ScheduleService {
scheduleManager.addOrUpdateCronJob(new JobKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
new TriggerKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
- TestJob.class,// TODO
- datasetTableTask.getCron(), new Date(datasetTableTask.getStartTime()), endTime);
+ ExtractDataJob.class,
+ datasetTableTask.getCron(), new Date(datasetTableTask.getStartTime()), endTime,
+ scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId()));
}
}
diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
index 6bd40b6720..a74b955d40 100644
--- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
+++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
@@ -71,7 +71,9 @@ public class DataSetTableService {
public List list(DataSetTableRequest dataSetTableRequest) {
DatasetTableExample datasetTableExample = new DatasetTableExample();
- datasetTableExample.createCriteria().andSceneIdEqualTo(dataSetTableRequest.getSceneId());
+ if(StringUtils.isNotEmpty(dataSetTableRequest.getSceneId())){
+ datasetTableExample.createCriteria().andSceneIdEqualTo(dataSetTableRequest.getSceneId());
+ }
if (StringUtils.isNotEmpty(dataSetTableRequest.getSort())) {
datasetTableExample.setOrderByClause(dataSetTableRequest.getSort());
}
@@ -92,7 +94,7 @@ public class DataSetTableService {
}
public Map> getFieldsFromDE(DataSetTableRequest dataSetTableRequest) throws Exception {
- DatasetTableField datasetTableField = new DatasetTableField();
+ DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(dataSetTableRequest.getId());
datasetTableField.setChecked(Boolean.TRUE);
List fields = dataSetTableFieldsService.list(datasetTableField);
@@ -121,7 +123,7 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds);
String table = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class).getTable();
- DatasetTableField datasetTableField = new DatasetTableField();
+ DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(dataSetTableRequest.getId());
datasetTableField.setChecked(Boolean.TRUE);
List fields = dataSetTableFieldsService.list(datasetTableField);
@@ -137,15 +139,13 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class).getTable();
-// datasourceRequest.setTable(table);
- DatasetTableField datasetTableField = new DatasetTableField();
+ DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(dataSetTableRequest.getId());
datasetTableField.setChecked(Boolean.TRUE);
List fields = dataSetTableFieldsService.list(datasetTableField);
String[] fieldArray = fields.stream().map(DatasetTableField::getOriginName).toArray(String[]::new);
-// datasourceRequest.setQuery("SELECT " + StringUtils.join(fieldArray, ",") + " FROM " + table + " LIMIT 0,10;");
datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fieldArray) + " LIMIT 0,10");
List data = new ArrayList<>();
@@ -154,17 +154,6 @@ public class DataSetTableService {
} catch (Exception e) {
}
-
- /*JSONArray jsonArray = new JSONArray();
- if (CollectionUtils.isNotEmpty(data)) {
- data.forEach(ele -> {
- JSONObject jsonObject = new JSONObject();
- for (int i = 0; i < ele.length; i++) {
- jsonObject.put(fieldArray[i], ele[i]);
- }
- jsonArray.add(jsonObject);
- });
- }*/
List