center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat: 增量更新数据

Browse Source
This commit is contained in:
TaoJinlong 2021-03-18 17:59:52 +08:00
parent 5f823af95a
commit e97297fd84
20 changed files with 1153 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/pom.xml
View File

@ -326,6 +326,11 @@
<artifactId>hbase-common</artifactId>
<version>2.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-mapreduce</artifactId>
<version>2.4.1</version>
</dependency>
<dependency>
<groupId>org.testng</groupId>

17
backend/src/main/java/io/dataease/base/domain/DatasetTableIncrementalConfig.java Normal file
View File

@ -0,0 +1,17 @@
package io.dataease.base.domain;
import java.io.Serializable;
import lombok.Data;
@Data
public class DatasetTableIncrementalConfig implements Serializable {
private String id;
private String tableId;
private String incrementalDelete;
private String incrementalAdd;
private static final long serialVersionUID = 1L;
}

480
backend/src/main/java/io/dataease/base/domain/DatasetTableIncrementalConfigExample.java Normal file
View File

@ -0,0 +1,480 @@
package io.dataease.base.domain;
import java.util.ArrayList;
import java.util.List;
public class DatasetTableIncrementalConfigExample {
protected String orderByClause;
protected boolean distinct;
protected List<Criteria> oredCriteria;
public DatasetTableIncrementalConfigExample() {
oredCriteria = new ArrayList<Criteria>();
}
public void setOrderByClause(String orderByClause) {
this.orderByClause = orderByClause;
}
public String getOrderByClause() {
return orderByClause;
}
public void setDistinct(boolean distinct) {
this.distinct = distinct;
}
public boolean isDistinct() {
return distinct;
}
public List<Criteria> getOredCriteria() {
return oredCriteria;
}
public void or(Criteria criteria) {
oredCriteria.add(criteria);
}
public Criteria or() {
Criteria criteria = createCriteriaInternal();
oredCriteria.add(criteria);
return criteria;
}
public Criteria createCriteria() {
Criteria criteria = createCriteriaInternal();
if (oredCriteria.size() == 0) {
oredCriteria.add(criteria);
}
return criteria;
}
protected Criteria createCriteriaInternal() {
Criteria criteria = new Criteria();
return criteria;
}
public void clear() {
oredCriteria.clear();
orderByClause = null;
distinct = false;
}
protected abstract static class GeneratedCriteria {
protected List<Criterion> criteria;
protected GeneratedCriteria() {
super();
criteria = new ArrayList<Criterion>();
}
public boolean isValid() {
return criteria.size() > 0;
}
public List<Criterion> getAllCriteria() {
return criteria;
}
public List<Criterion> getCriteria() {
return criteria;
}
protected void addCriterion(String condition) {
if (condition == null) {
throw new RuntimeException("Value for condition cannot be null");
}
criteria.add(new Criterion(condition));
}
protected void addCriterion(String condition, Object value, String property) {
if (value == null) {
throw new RuntimeException("Value for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value));
}
protected void addCriterion(String condition, Object value1, Object value2, String property) {
if (value1 == null || value2 == null) {
throw new RuntimeException("Between values for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value1, value2));
}
public Criteria andIdIsNull() {
addCriterion("id is null");
return (Criteria) this;
}
public Criteria andIdIsNotNull() {
addCriterion("id is not null");
return (Criteria) this;
}
public Criteria andIdEqualTo(String value) {
addCriterion("id =", value, "id");
return (Criteria) this;
}
public Criteria andIdNotEqualTo(String value) {
addCriterion("id <>", value, "id");
return (Criteria) this;
}
public Criteria andIdGreaterThan(String value) {
addCriterion("id >", value, "id");
return (Criteria) this;
}
public Criteria andIdGreaterThanOrEqualTo(String value) {
addCriterion("id >=", value, "id");
return (Criteria) this;
}
public Criteria andIdLessThan(String value) {
addCriterion("id <", value, "id");
return (Criteria) this;
}
public Criteria andIdLessThanOrEqualTo(String value) {
addCriterion("id <=", value, "id");
return (Criteria) this;
}
public Criteria andIdLike(String value) {
addCriterion("id like", value, "id");
return (Criteria) this;
}
public Criteria andIdNotLike(String value) {
addCriterion("id not like", value, "id");
return (Criteria) this;
}
public Criteria andIdIn(List<String> values) {
addCriterion("id in", values, "id");
return (Criteria) this;
}
public Criteria andIdNotIn(List<String> values) {
addCriterion("id not in", values, "id");
return (Criteria) this;
}
public Criteria andIdBetween(String value1, String value2) {
addCriterion("id between", value1, value2, "id");
return (Criteria) this;
}
public Criteria andIdNotBetween(String value1, String value2) {
addCriterion("id not between", value1, value2, "id");
return (Criteria) this;
}
public Criteria andTableIdIsNull() {
addCriterion("table_id is null");
return (Criteria) this;
}
public Criteria andTableIdIsNotNull() {
addCriterion("table_id is not null");
return (Criteria) this;
}
public Criteria andTableIdEqualTo(String value) {
addCriterion("table_id =", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdNotEqualTo(String value) {
addCriterion("table_id <>", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdGreaterThan(String value) {
addCriterion("table_id >", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdGreaterThanOrEqualTo(String value) {
addCriterion("table_id >=", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdLessThan(String value) {
addCriterion("table_id <", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdLessThanOrEqualTo(String value) {
addCriterion("table_id <=", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdLike(String value) {
addCriterion("table_id like", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdNotLike(String value) {
addCriterion("table_id not like", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdIn(List<String> values) {
addCriterion("table_id in", values, "tableId");
return (Criteria) this;
}
public Criteria andTableIdNotIn(List<String> values) {
addCriterion("table_id not in", values, "tableId");
return (Criteria) this;
}
public Criteria andTableIdBetween(String value1, String value2) {
addCriterion("table_id between", value1, value2, "tableId");
return (Criteria) this;
}
public Criteria andTableIdNotBetween(String value1, String value2) {
addCriterion("table_id not between", value1, value2, "tableId");
return (Criteria) this;
}
public Criteria andIncrementalDeleteIsNull() {
addCriterion("incremental_delete is null");
return (Criteria) this;
}
public Criteria andIncrementalDeleteIsNotNull() {
addCriterion("incremental_delete is not null");
return (Criteria) this;
}
public Criteria andIncrementalDeleteEqualTo(String value) {
addCriterion("incremental_delete =", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteNotEqualTo(String value) {
addCriterion("incremental_delete <>", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteGreaterThan(String value) {
addCriterion("incremental_delete >", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteGreaterThanOrEqualTo(String value) {
addCriterion("incremental_delete >=", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteLessThan(String value) {
addCriterion("incremental_delete <", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteLessThanOrEqualTo(String value) {
addCriterion("incremental_delete <=", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteLike(String value) {
addCriterion("incremental_delete like", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteNotLike(String value) {
addCriterion("incremental_delete not like", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteIn(List<String> values) {
addCriterion("incremental_delete in", values, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteNotIn(List<String> values) {
addCriterion("incremental_delete not in", values, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteBetween(String value1, String value2) {
addCriterion("incremental_delete between", value1, value2, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteNotBetween(String value1, String value2) {
addCriterion("incremental_delete not between", value1, value2, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalAddIsNull() {
addCriterion("incremental_add is null");
return (Criteria) this;
}
public Criteria andIncrementalAddIsNotNull() {
addCriterion("incremental_add is not null");
return (Criteria) this;
}
public Criteria andIncrementalAddEqualTo(String value) {
addCriterion("incremental_add =", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddNotEqualTo(String value) {
addCriterion("incremental_add <>", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddGreaterThan(String value) {
addCriterion("incremental_add >", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddGreaterThanOrEqualTo(String value) {
addCriterion("incremental_add >=", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddLessThan(String value) {
addCriterion("incremental_add <", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddLessThanOrEqualTo(String value) {
addCriterion("incremental_add <=", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddLike(String value) {
addCriterion("incremental_add like", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddNotLike(String value) {
addCriterion("incremental_add not like", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddIn(List<String> values) {
addCriterion("incremental_add in", values, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddNotIn(List<String> values) {
addCriterion("incremental_add not in", values, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddBetween(String value1, String value2) {
addCriterion("incremental_add between", value1, value2, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddNotBetween(String value1, String value2) {
addCriterion("incremental_add not between", value1, value2, "incrementalAdd");
return (Criteria) this;
}
}
public static class Criteria extends GeneratedCriteria {
protected Criteria() {
super();
}
}
public static class Criterion {
private String condition;
private Object value;
private Object secondValue;
private boolean noValue;
private boolean singleValue;
private boolean betweenValue;
private boolean listValue;
private String typeHandler;
public String getCondition() {
return condition;
}
public Object getValue() {
return value;
}
public Object getSecondValue() {
return secondValue;
}
public boolean isNoValue() {
return noValue;
}
public boolean isSingleValue() {
return singleValue;
}
public boolean isBetweenValue() {
return betweenValue;
}
public boolean isListValue() {
return listValue;
}
public String getTypeHandler() {
return typeHandler;
}
protected Criterion(String condition) {
super();
this.condition = condition;
this.typeHandler = null;
this.noValue = true;
}
protected Criterion(String condition, Object value, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.typeHandler = typeHandler;
if (value instanceof List<?>) {
this.listValue = true;
} else {
this.singleValue = true;
}
}
protected Criterion(String condition, Object value) {
this(condition, value, null);
}
protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.secondValue = secondValue;
this.typeHandler = typeHandler;
this.betweenValue = true;
}
protected Criterion(String condition, Object value, Object secondValue) {
this(condition, value, secondValue, null);
}
}
}

22
backend/src/main/java/io/dataease/base/mapper/DatasetTableIncrementalConfigMapper.java Normal file
View File

@ -0,0 +1,22 @@
package io.dataease.base.mapper;
import io.dataease.base.domain.DatasetTableIncrementalConfig;
import io.dataease.base.domain.DatasetTableIncrementalConfigExample;
import java.util.List;
import org.apache.ibatis.annotations.Param;
public interface DatasetTableIncrementalConfigMapper {
long countByExample(DatasetTableIncrementalConfigExample example);
int deleteByExample(DatasetTableIncrementalConfigExample example);
int insert(DatasetTableIncrementalConfig record);
int insertSelective(DatasetTableIncrementalConfig record);
List<DatasetTableIncrementalConfig> selectByExample(DatasetTableIncrementalConfigExample example);
int updateByExampleSelective(@Param("record") DatasetTableIncrementalConfig record, @Param("example") DatasetTableIncrementalConfigExample example);
int updateByExample(@Param("record") DatasetTableIncrementalConfig record, @Param("example") DatasetTableIncrementalConfigExample example);
}

164
backend/src/main/java/io/dataease/base/mapper/DatasetTableIncrementalConfigMapper.xml Normal file
View File

@ -0,0 +1,164 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="io.dataease.base.mapper.DatasetTableIncrementalConfigMapper">
<resultMap id="BaseResultMap" type="io.dataease.base.domain.DatasetTableIncrementalConfig">
<result column="id" jdbcType="VARCHAR" property="id" />
<result column="table_id" jdbcType="VARCHAR" property="tableId" />
<result column="incremental_delete" jdbcType="VARCHAR" property="incrementalDelete" />
<result column="incremental_add" jdbcType="VARCHAR" property="incrementalAdd" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
<foreach collection="oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Update_By_Example_Where_Clause">
<where>
<foreach collection="example.oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Base_Column_List">
id, table_id, incremental_delete, incremental_add
</sql>
<select id="selectByExample" parameterType="io.dataease.base.domain.DatasetTableIncrementalConfigExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
from dataset_table_incremental_config
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<delete id="deleteByExample" parameterType="io.dataease.base.domain.DatasetTableIncrementalConfigExample">
delete from dataset_table_incremental_config
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</delete>
<insert id="insert" parameterType="io.dataease.base.domain.DatasetTableIncrementalConfig">
insert into dataset_table_incremental_config (id, table_id, incremental_delete,
incremental_add)
values (#{id,jdbcType=VARCHAR}, #{tableId,jdbcType=VARCHAR}, #{incrementalDelete,jdbcType=VARCHAR},
#{incrementalAdd,jdbcType=VARCHAR})
</insert>
<insert id="insertSelective" parameterType="io.dataease.base.domain.DatasetTableIncrementalConfig">
insert into dataset_table_incremental_config
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="id != null">
id,
</if>
<if test="tableId != null">
table_id,
</if>
<if test="incrementalDelete != null">
incremental_delete,
</if>
<if test="incrementalAdd != null">
incremental_add,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
#{id,jdbcType=VARCHAR},
</if>
<if test="tableId != null">
#{tableId,jdbcType=VARCHAR},
</if>
<if test="incrementalDelete != null">
#{incrementalDelete,jdbcType=VARCHAR},
</if>
<if test="incrementalAdd != null">
#{incrementalAdd,jdbcType=VARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="io.dataease.base.domain.DatasetTableIncrementalConfigExample" resultType="java.lang.Long">
select count(*) from dataset_table_incremental_config
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</select>
<update id="updateByExampleSelective" parameterType="map">
update dataset_table_incremental_config
<set>
<if test="record.id != null">
id = #{record.id,jdbcType=VARCHAR},
</if>
<if test="record.tableId != null">
table_id = #{record.tableId,jdbcType=VARCHAR},
</if>
<if test="record.incrementalDelete != null">
incremental_delete = #{record.incrementalDelete,jdbcType=VARCHAR},
</if>
<if test="record.incrementalAdd != null">
incremental_add = #{record.incrementalAdd,jdbcType=VARCHAR},
</if>
</set>
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
<update id="updateByExample" parameterType="map">
update dataset_table_incremental_config
set id = #{record.id,jdbcType=VARCHAR},
table_id = #{record.tableId,jdbcType=VARCHAR},
incremental_delete = #{record.incrementalDelete,jdbcType=VARCHAR},
incremental_add = #{record.incrementalAdd,jdbcType=VARCHAR}
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
</mapper>

2
backend/src/main/java/io/dataease/commons/constants/ScheduleType.java
View File

@ -1,5 +1,5 @@
package io.dataease.commons.constants;
public enum ScheduleType {
CRON, SIMPLE
CRON, SIMPLE, SIMPLE_COMPLETE
}

5
backend/src/main/java/io/dataease/commons/constants/UpdateType.java Normal file
View File

@ -0,0 +1,5 @@
package io.dataease.commons.constants;
public enum UpdateType {
all_scope, add_scope
}

12
backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java
View File

@ -2,6 +2,7 @@ package io.dataease.controller.dataset;
import io.dataease.base.domain.DatasetTable;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableIncrementalConfig;
import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.datasource.dto.TableFiled;
import io.dataease.service.dataset.DataSetTableService;
@ -70,4 +71,15 @@ public class DataSetTableController {
public Map<String, Object> getSQLPreview(@RequestBody DataSetTableRequest dataSetTableRequest) throws Exception {
return dataSetTableService.getSQLPreview(dataSetTableRequest);
}
@PostMapping("incrementalConfig")
public DatasetTableIncrementalConfig incrementalConfig(@RequestBody DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception {
return dataSetTableService.incrementalConfig(datasetTableIncrementalConfig);
}
@PostMapping("save/incrementalConfig")
public void saveIncrementalConfig(@RequestBody DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception {
dataSetTableService.saveIncrementalConfig(datasetTableIncrementalConfig);
}
}

2
backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java
View File

@ -8,6 +8,7 @@ public abstract class DeScheduleJob implements Job {
protected String datasetTableId;
protected String expression;
protected String taskId;
protected String updateType;
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
@ -16,6 +17,7 @@ public abstract class DeScheduleJob implements Job {
this.datasetTableId = jobDataMap.getString("datasetTableId");
this.expression = jobDataMap.getString("expression");
this.taskId = jobDataMap.getString("taskId");
this.updateType = jobDataMap.getString("updateType");
LogUtil.info(jobKey.getGroup() + " Running: " + datasetTableId);
LogUtil.info("CronExpression: " + expression);

2
backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java
View File

@ -16,7 +16,7 @@ public class ExtractDataJob extends DeScheduleJob{
@Override
void businessExecute(JobExecutionContext context) {
extractDataService.extractData(datasetTableId, taskId);
extractDataService.extractData(datasetTableId, taskId, updateType);
}
}

3
backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java
View File

@ -369,11 +369,12 @@ public class ScheduleManager {
addOrUpdateCronJob(jobKey, triggerKey, jobClass, cron, startTime, endTime, null);
}
public JobDataMap getDefaultJobDataMap(String resourceId, String expression, String taskId) {
public JobDataMap getDefaultJobDataMap(String resourceId, String expression, String taskId, String updateType) {
JobDataMap jobDataMap = new JobDataMap();
jobDataMap.put("datasetTableId", resourceId);
jobDataMap.put("taskId", taskId);
jobDataMap.put("expression", expression);
jobDataMap.put("updateType", updateType);
return jobDataMap;
}

10
backend/src/main/java/io/dataease/service/ScheduleService.java
View File

@ -1,6 +1,7 @@
package io.dataease.service;
import io.dataease.base.domain.DatasetTableTask;
import io.dataease.commons.constants.ScheduleType;
import io.dataease.job.sechedule.ExtractDataJob;
import io.dataease.job.sechedule.ScheduleManager;
import org.apache.commons.lang3.StringUtils;
@ -21,12 +22,13 @@ public class ScheduleService {
private ScheduleManager scheduleManager;
public void addSchedule(DatasetTableTask datasetTableTask) throws Exception {
if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), "0")) {
if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), ScheduleType.SIMPLE.toString())) {
scheduleManager.addOrUpdateSingleJob(new JobKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
new TriggerKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
ExtractDataJob.class,
new Date(datasetTableTask.getStartTime()), scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId()));
} else if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), "1")) {
new Date(datasetTableTask.getStartTime()),
scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId(), datasetTableTask.getType()));
} else if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), ScheduleType.CRON.toString())) {
Date endTime;
if (datasetTableTask.getEndTime() == null || datasetTableTask.getEndTime() == 0) {
endTime = null;
@ -38,7 +40,7 @@ public class ScheduleService {
new TriggerKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
ExtractDataJob.class,
datasetTableTask.getCron(), new Date(datasetTableTask.getStartTime()), endTime,
scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId()));
scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId(), datasetTableTask.getType()));
}
}

53
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -2,10 +2,8 @@ package io.dataease.service.dataset;
import com.google.gson.Gson;
import io.dataease.base.domain.DatasetTable;
import io.dataease.base.domain.DatasetTableExample;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.Datasource;
import io.dataease.base.domain.*;
import io.dataease.base.mapper.DatasetTableIncrementalConfigMapper;
import io.dataease.base.mapper.DatasetTableMapper;
import io.dataease.base.mapper.DatasourceMapper;
import io.dataease.commons.utils.BeanUtils;
@ -40,6 +38,8 @@ public class DataSetTableService {
private DataSetTableFieldsService dataSetTableFieldsService;
@Resource
private DataSetTableTaskService dataSetTableTaskService;
@Resource
private DatasetTableIncrementalConfigMapper datasetTableIncrementalConfigMapper;
public void batchInsert(List<DatasetTable> datasetTable) throws Exception {
for (DatasetTable table : datasetTable) {
@ -261,6 +261,20 @@ public class DataSetTableService {
return data;
}
public List<String[]> getDataSetDataBySql(String datasourceId, String table, String sql) {
List<String[]> data = new ArrayList<>();
Datasource ds = datasourceMapper.selectByPrimaryKey(datasourceId);
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
datasourceRequest.setQuery(sql);
try {
return datasourceProvider.getData(datasourceRequest);
} catch (Exception e) {
}
return data;
}
public void saveTableField(DatasetTable datasetTable) throws Exception {
Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
DataSetTableRequest dataSetTableRequest = new DataSetTableRequest();
@ -349,4 +363,35 @@ public class DataSetTableService {
return 0;
}
}
public DatasetTableIncrementalConfig incrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig){
if(StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())){return new DatasetTableIncrementalConfig();}
DatasetTableIncrementalConfigExample example = new DatasetTableIncrementalConfigExample();
example.createCriteria().andTableIdEqualTo(datasetTableIncrementalConfig.getTableId());
List<DatasetTableIncrementalConfig> configs = datasetTableIncrementalConfigMapper.selectByExample(example);
if(CollectionUtils.isNotEmpty(configs)){
return configs.get(0);
}else {
return new DatasetTableIncrementalConfig();
}
}
public DatasetTableIncrementalConfig incrementalConfig(String datasetTableId){
DatasetTableIncrementalConfig datasetTableIncrementalConfig = new DatasetTableIncrementalConfig();
datasetTableIncrementalConfig.setTableId(datasetTableId);
return incrementalConfig(datasetTableIncrementalConfig);
}
public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig){
if(StringUtils.isEmpty(datasetTableIncrementalConfig.getId())){
datasetTableIncrementalConfig.setId(UUID.randomUUID().toString());
datasetTableIncrementalConfigMapper.insertSelective(datasetTableIncrementalConfig);
}else{
DatasetTableIncrementalConfigExample example = new DatasetTableIncrementalConfigExample();
example.createCriteria().andTableIdEqualTo(datasetTableIncrementalConfig.getTableId());
datasetTableIncrementalConfigMapper.updateByExample(datasetTableIncrementalConfig, example);
}
}
}

4
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
View File

@ -70,6 +70,10 @@ public class DataSetTableTaskService {
return datasetTableTaskMapper.selectByPrimaryKey(id);
}
public void update(DatasetTableTask datasetTableTask) {
datasetTableTaskMapper.updateByPrimaryKey(datasetTableTask);
}
public List<DatasetTableTask> list(DatasetTableTask datasetTableTask) {
DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample();
DatasetTableTaskExample.Criteria criteria = datasetTableTaskExample.createCriteria();

206
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -1,19 +1,23 @@
package io.dataease.service.dataset;
import com.google.gson.Gson;
import io.dataease.base.domain.DatasetTable;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableTaskLog;
import io.dataease.base.domain.*;
import io.dataease.commons.constants.JobStatus;
import io.dataease.commons.constants.ScheduleType;
import io.dataease.commons.constants.UpdateType;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.commons.utils.LogUtil;
import io.dataease.dto.dataset.DataSetTaskLogDTO;
import io.dataease.dto.dataset.DataTableInfoDTO;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.security.MessageDigest;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
@ -28,54 +32,74 @@ public class ExtractDataService {
private DataSetTableFieldsService dataSetTableFieldsService;
@Resource
private DataSetTableTaskLogService dataSetTableTaskLogService;
@Resource
private DataSetTableTaskService dataSetTableTaskService;
private Long pageSize = 10000l;
private static ExecutorService pool = Executors.newScheduledThreadPool(50); //设置连接池
private Connection connection;
private static String lastUpdateTime = "${__last_update_time__}";
private static String currentUpdateTime = "${__current_update_time__}";
private static String column_family = "dataease";
public void extractData(String datasetTableId, String taskId) {
public void extractData(String datasetTableId, String taskId, String type) {
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
UpdateType updateType = UpdateType.valueOf(type);
try {
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
Admin admin = getConnection().getAdmin();
DatasetTable datasetTable = dataSetTableService.get(datasetTableId);
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build());
String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
TableName tableName = TableName.valueOf(table + "-" + datasetTable.getDataSourceId());
if(!admin.tableExists(tableName)){
TableDescriptorBuilder descBuilder = TableDescriptorBuilder.newBuilder(tableName);
ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.of("cf");
descBuilder.setColumnFamily(hcd);
TableDescriptor desc = descBuilder.build();
admin.createTable(desc);
}
admin.disableTable(tableName);
admin.truncateTable(tableName, true);
Table tab = getConnection().getTable(tableName);
Long total = dataSetTableService.getDataSetTotalData(datasetTable.getDataSourceId(), table);
Long pageCount = total % pageSize == 0 ? total / pageSize : (total / pageSize) + 1;
for (Long pageIndex = 1l; pageIndex <= pageCount; pageIndex++) {
List<String[]> data = dataSetTableService.getDataSetPageData(datasetTable.getDataSourceId(), table, datasetTableFields, pageIndex, pageSize);
for (String[] d : data) {
for(int i=0;i<datasetTableFields.size();i++){
Put put = new Put(UUID.randomUUID().toString().getBytes());
String value = d[i];
if(value == null){
value = "null";
}
put.addColumn("cf".getBytes(), datasetTableFields.get(i).getOriginName().getBytes(), value.getBytes());
tab.put(put);
switch (updateType){
// 全量更新
case all_scope:
writeDatasetTableTaskLog(datasetTableTaskLog,datasetTableId, taskId);
if(!admin.tableExists(tableName)){
creatHaseTable(tableName, admin);
}
}
extractAllData(admin, tableName, table, datasetTable, datasetTableFields);
datasetTableTaskLog.setStatus(JobStatus.Completed.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
break;
case add_scope:
// 增量更新
if(!admin.tableExists(tableName)){
LogUtil.error("TableName error, dataaset: " + datasetTableId);
return;
}
DatasetTableIncrementalConfig datasetTableIncrementalConfig = dataSetTableService.incrementalConfig(datasetTableId);
if(datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())){
return;
}
DatasetTableTaskLog request = new DatasetTableTaskLog();
request.setTableId(datasetTableId);
request.setStatus(JobStatus.Completed.name());
List<DataSetTaskLogDTO> dataSetTaskLogDTOS = dataSetTableTaskLogService.list(request);
if(CollectionUtils.isEmpty(dataSetTaskLogDTOS)){
return;
}
writeDatasetTableTaskLog(datasetTableTaskLog,datasetTableId, taskId);
// 增量添加
if(StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd())){
String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, dataSetTaskLogDTOS.get(0).getStartTime().toString()
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()));
extractIncrementalData(tableName,table,datasetTable, datasetTableFields, sql, "add");
}
// 增量删除
if( StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete())){
String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, dataSetTaskLogDTOS.get(0).getStartTime().toString()
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()));
extractIncrementalData(tableName,table,datasetTable, datasetTableFields, sql, "delete");
}
datasetTableTaskLog.setStatus(JobStatus.Completed.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
break;
}
datasetTableTaskLog.setStatus(JobStatus.Completed.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}catch (Exception e){
e.printStackTrace();
LogUtil.error("ExtractData error, dataaset: " + datasetTableId);
@ -84,8 +108,75 @@ public class ExtractDataService {
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
finally {
DatasetTableTask datasetTableTask = dataSetTableTaskService.get(taskId);
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())){
datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString());
dataSetTableTaskService.update(datasetTableTask);
}
}
}
private void writeDatasetTableTaskLog(DatasetTableTaskLog datasetTableTaskLog, String datasetTableId, String taskId){
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
private void creatHaseTable(TableName tableName, Admin admin)throws Exception{
TableDescriptorBuilder descBuilder = TableDescriptorBuilder.newBuilder(tableName);
ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.of(column_family);
descBuilder.setColumnFamily(hcd);
TableDescriptor desc = descBuilder.build();
admin.createTable(desc);
}
private void extractAllData(Admin admin, TableName tableName, String table, DatasetTable datasetTable, List<DatasetTableField> datasetTableFields)throws Exception{
admin.disableTable(tableName);
admin.truncateTable(tableName, true);
Table tab = getConnection().getTable(tableName);
Long total = dataSetTableService.getDataSetTotalData(datasetTable.getDataSourceId(), table);
Long pageCount = total % pageSize == 0 ? total / pageSize : (total / pageSize) + 1;
for (Long pageIndex = 1l; pageIndex <= pageCount; pageIndex++) {
List<String[]> data = dataSetTableService.getDataSetPageData(datasetTable.getDataSourceId(), table, datasetTableFields, pageIndex, pageSize);
insertDataToHbaseTable(data,datasetTableFields,tab);
}
}
private void extractIncrementalData(TableName tableName, String table, DatasetTable datasetTable, List<DatasetTableField> datasetTableFields, String sql, String type)throws Exception{
Table tab = getConnection().getTable(tableName);
List<String[]> data = dataSetTableService.getDataSetDataBySql(datasetTable.getDataSourceId(), table, sql);
if (type.equalsIgnoreCase("add")){
insertDataToHbaseTable(data,datasetTableFields,tab);
}else {
deleteDataFromHbaseTable(data,datasetTableFields,tab);
}
}
private void insertDataToHbaseTable(List<String[]> data, List<DatasetTableField> datasetTableFields, Table tab)throws Exception{
for (String[] d : data) {
Put put = new Put(md5(generateStr(datasetTableFields.size(), d)).getBytes());
for(int i=0;i<datasetTableFields.size();i++){
String value = d[i];
if(value == null){
value = "null";
}
put.addColumn(column_family.getBytes(), datasetTableFields.get(i).getOriginName().getBytes(), value.getBytes());
}
tab.put(put);
}
}
private void deleteDataFromHbaseTable(List<String[]> data, List<DatasetTableField> datasetTableFields, Table tab)throws Exception{
for (String[] d : data) {
Delete delete = new Delete(md5(generateStr(datasetTableFields.size(), d)).getBytes());
tab.delete(delete);
}
}
private synchronized Connection getConnection() throws Exception{
if(connection == null || connection.isClosed()){
@ -94,4 +185,43 @@ public class ExtractDataService {
}
return connection;
}
private static final char[] HEX_DIGITS = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
private static final String UTF_8 = "UTF-8";
public static String md5(String src) {
return md5(src, UTF_8);
}
public static String md5(String src, String charset) {
try {
byte[] strTemp = io.micrometer.core.instrument.util.StringUtils.isEmpty(charset) ? src.getBytes() : src.getBytes(charset);
MessageDigest mdTemp = MessageDigest.getInstance("MD5");
mdTemp.update(strTemp);
byte[] md = mdTemp.digest();
int j = md.length;
char[] str = new char[j * 2];
int k = 0;
for (byte byte0 : md) {
str[k++] = HEX_DIGITS[byte0 >>> 4 & 0xf];
str[k++] = HEX_DIGITS[byte0 & 0xf];
}
return new String(str);
} catch (Exception e) {
throw new RuntimeException("MD5 encrypt error:", e);
}
}
public String generateStr(int size, String[] d ){
String str = null;
for(int i=0;i<size;i++){
str = str + d[i];
}
return str;
}
}

10
backend/src/main/resources/db/migration/V13__dataset_tables_incre.sql Normal file
View File

@ -0,0 +1,10 @@
DROP TABLE IF EXISTS `dataset_table_incremental_config`;
CREATE TABLE IF NOT EXISTS `dataset_table_incremental_config`
(
`id` varchar(50) NOT NULL COMMENT 'ID',
`table_id` varchar(50) NOT NULL COMMENT '表ID',
`incremental_delete` longtext COMMENT '详细信息',
`incremental_add` longtext COMMENT '详细信息',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4;

19
backend/src/main/resources/generatorConfig.xml
View File

@ -37,11 +37,11 @@
</commentGenerator>
<!-- jdbc连接信息 --> <!-- EduLoanManage EduTestDataBase -->
<!--<jdbcConnection driverClass="com.mysql.jdbc.Driver" connectionURL="jdbc:mysql://192.168.20.180:3306/fit2cloud"-->
<!--userId="root" password="Fit2cloud2015!" />-->
<jdbcConnection driverClass="com.mysql.cj.jdbc.Driver"
connectionURL="${spring.datasource.url}&amp;nullCatalogMeansCurrent=true"
userId="${spring.datasource.username}" password="${spring.datasource.password}"/>
<jdbcConnection driverClass="com.mysql.jdbc.Driver" connectionURL="jdbc:mysql://62.234.205.170:3306/dataease"
userId="root" password="Password123@mysql" />
<!-- <jdbcConnection driverClass="com.mysql.cj.jdbc.Driver"-->
<!-- connectionURL="${spring.datasource.url}&amp;nullCatalogMeansCurrent=true"-->
<!-- userId="${spring.datasource.username}" password="${spring.datasource.password}"/>-->
<!-- javaTypeResolver式类型转换的信息 -->
<javaTypeResolver>
@ -64,8 +64,13 @@
<!--要生成的数据库表 -->
<!-- <table tableName="datasource"/>-->
<table tableName="panel_group"/>
<table tableName="dataset_table_incremental_config" >
<!--以下为添加内容 -->
<columnOverride column="incremental_delete" javaType="java.lang.String" jdbcType="VARCHAR" />
<columnOverride column="incremental_add" javaType="java.lang.String" jdbcType="VARCHAR" />
</table>
</context>

8
frontend/src/lang/zh.js
View File

@ -671,7 +671,12 @@ export default {
add_sql_table: '添加SQL',
preview: '预览',
pls_input_name: '请输入名称',
connect_mode: '连接模式'
connect_mode: '连接模式',
incremental_update_type: '增量更新方式:',
incremental_add: '增量添加:',
incremental_delete: '增量删除:',
last_update_time: '上次更新时间:',
current_update_time: '当前更新时间:'
},
datasource: {
create: '新建数据连接',
@ -689,6 +694,7 @@ export default {
please_input_port: '请输入端口',
modify: '编辑数据连接',
validate_success: '校验成功',
validate: '校验',
delete: '删除组织',
delete_confirm: '删除该组织会关联删除该组织下的所有资源(如：相关工作空间，项目，测试用例等），确定要删除吗?',
input_name: '请输入名称',

211
frontend/src/views/dataset/data/UpdateInfo.vue
View File

@ -79,12 +79,11 @@
<el-select v-model="taskForm.type" size="mini">
<el-option
:label="$t('dataset.all_scope')"
value="0"
value="all_scope"
/>
<el-option
:label="$t('dataset.add_scope')"
value="1"
:disabled="true"
value="add_scope"
/>
</el-select>
</el-form-item>
@ -97,22 +96,22 @@
/>
</el-form-item>
<el-form-item :label="$t('dataset.execute_rate')" prop="rate">
<el-select v-model="taskForm.rate" size="mini" @change="onRateChange">
<el-select v-model="taskForm.rate" @change="onRateChange">
<el-option
:label="$t('dataset.execute_once')"
value="0"
value="SIMPLE"
/>
<el-option
:label="$t('dataset.cron_config')"
value="1"
value="CRON"
/>
</el-select>
</el-form-item>
<el-form-item v-if="taskForm.rate === '1'" label="">
<el-form-item v-if="taskForm.rate === 'CRON'" label="">
<el-input v-model="taskForm.cron" size="mini" style="width: 50%" />
</el-form-item>
<el-form-item :label="$t('dataset.end_time')" prop="end">
<el-select v-model="taskForm.end" size="mini" :disabled="taskForm.rate === '0'">
<el-select v-model="taskForm.end" size="mini">
<el-option
:label="$t('dataset.no_limit')"
value="0"
@ -163,8 +162,9 @@
:label="$t('dataset.execute_rate')"
>
<template slot-scope="scope">
<span v-if="scope.row.rate === '0'">{{ $t('dataset.execute_once') }}</span>
<span v-if="scope.row.rate === '1'">{{ $t('dataset.cron_config') }}</span>
<span v-if="scope.row.rate === 'SIMPLE'">{{ $t('dataset.execute_once') }}</span>
<span v-if="scope.row.rate === 'SIMPLE_COMPLETE'">{{ $t('dataset.execute_once') }}</span>
<span v-if="scope.row.rate === 'CRON'">{{ $t('dataset.cron_config') }}</span>
</template>
</el-table-column>
<el-table-column
@ -176,6 +176,7 @@
type="primary"
icon="el-icon-edit"
circle
:disabled="scope.row.rate === 'SIMPLE_COMPLETE'"
@click="addTask(scope.row)"
/>
<el-button
@ -190,10 +191,49 @@
</el-table>
</el-row>
<el-divider />
<el-row style="height: 26px;">
<el-row>
<el-col :span="6"><div>{{ $t('dataset.incremental_update_type') }}</div></el-col>
<el-col :span="18">
<el-radio-group v-model="incrementalUpdateType" size="small" @change="incrementalUpdateTypeChange">
<el-radio label="incrementalAdd" >{{ $t('dataset.incremental_add') }}</el-radio>
<el-radio label="incrementalDelete" >{{ $t('incremental_delete.incremental_update_type') }}</el-radio>
</el-radio-group>
</el-col>
</el-row>
</el-row>
<el-row style="height: 26px;">
<el-row>
<el-col :span="6" style="height: 26px;"><div style="height: 26px;">参数:</div></el-col>
<el-col :span="18">
<el-button type="text">{{ $t('dataset.last_update_time') }}</el-button>
<el-button type="text">{{ $t('dataset.current_update_time') }}</el-button>
</el-col>
</el-row>
</el-row>
<el-row>
<el-col style="min-width: 200px;">
<codemirror
ref="myCm"
v-model="sql"
class="codemirror"
:options="sqlOption"
@ready="onCmReady"
@focus="onCmFocus"
@input="onCmCodeChange"
/>
</el-col>
</el-row>
<div slot="footer" class="dialog-footer">
<!-- <el-button size="mini" @click="update_setting = false">{{ $t('dataset.cancel') }}</el-button>-->
<!-- <el-button type="primary" size="mini" @click="update_setting = false">{{ $t('dataset.confirm') }}</el-button>-->
<el-button size="mini" @click="update_setting = false">{{ $t('dataset.close') }}</el-button>
<el-button size="mini" @click="saveIncrementalConfig">{{ $t('dataset.confirm') }}</el-button>
</div>
</el-dialog>
</el-col>
@ -201,9 +241,33 @@
<script>
import { post } from '@/api/dataset/dataset'
import { codemirror } from 'vue-codemirror'
// 核心样式
import 'codemirror/lib/codemirror.css'
// 引入主题后还需要在 options 中指定主题才会生效
import 'codemirror/theme/solarized.css'
import 'codemirror/mode/sql/sql.js'
// require active-line.js
import 'codemirror/addon/selection/active-line.js'
// closebrackets
import 'codemirror/addon/edit/closebrackets.js'
// keyMap
import 'codemirror/mode/clike/clike.js'
import 'codemirror/addon/edit/matchbrackets.js'
import 'codemirror/addon/comment/comment.js'
import 'codemirror/addon/dialog/dialog.js'
import 'codemirror/addon/dialog/dialog.css'
import 'codemirror/addon/search/searchcursor.js'
import 'codemirror/addon/search/search.js'
import 'codemirror/keymap/emacs.js'
// 引入代码自动提示插件
import 'codemirror/addon/hint/show-hint.css'
import 'codemirror/addon/hint/sql-hint'
import 'codemirror/addon/hint/show-hint'
export default {
name: 'UpdateInfo',
components: { codemirror },
props: {
table: {
type: Object,
@ -216,9 +280,9 @@ export default {
update_task: false,
taskForm: {
name: '',
type: '0',
type: 'all_scope',
startTime: '',
rate: '0',
rate: 'SIMPLE',
cron: '',
endTime: '',
end: '0'
@ -246,7 +310,21 @@ export default {
end: [
{ required: true, message: this.$t('dataset.required'), trigger: 'change' }
]
}
},
sqlOption: {
tabSize: 2,
styleActiveLine: true,
lineNumbers: true,
line: true,
mode: 'text/x-sql',
theme: 'solarized',
hintOptions: { // 自定义提示选项
completeSingle: false // 当匹配只有一项的时候是否自动补全
}
},
incrementalUpdateType: 'incrementalAdd',
sql: '',
incrementalConfig: {}
}
},
watch: {
@ -255,10 +333,49 @@ export default {
this.listTaskLog()
}
},
computed: {
codemirror() {
return this.$refs.myCm.codemirror
}
},
mounted() {
window.onresize = () => {
return (() => {
this.height = window.innerHeight / 2
})()
}
this.height = window.innerHeight / 2
},
methods: {
incrementalUpdateTypeChange: function (){
if(this.incrementalUpdateType === 'incrementalAdd'){
if(this.sql){
this.incrementalConfig.incrementalDelete = this.sql
}
if(this.incrementalConfig.incrementalAdd){
this.sql = this.incrementalConfig.incrementalAdd
}else {
this.sql = ''
}
}
if(this.incrementalUpdateType === 'incrementalDelete'){
if(this.sql){
this.incrementalConfig.incrementalAdd = this.sql
}
if(this.incrementalConfig.incrementalDelete){
this.sql = this.incrementalConfig.incrementalDelete
}else {
this.sql = ''
}
}
},
showConfig() {
this.update_setting = true
this.listTask()
this.getIncrementalConfig()
},
addTask(task) {
if (!task) {
@ -275,6 +392,31 @@ export default {
this.taskData = response.data
})
},
getIncrementalConfig() {
post('/dataset/table/incrementalConfig', { tableId: this.table.id }).then(response => {
this.incrementalConfig = response.data
this.incrementalUpdateType = 'incrementalAdd'
if(this.incrementalConfig.incrementalAdd){
this.sql = this.incrementalConfig.incrementalAdd
}
})
},
saveIncrementalConfig() {
this.update_setting = false
if(this.incrementalUpdateType === 'incrementalAdd'){
this.incrementalConfig.incrementalAdd = this.sql
}else {
this.incrementalConfig.incrementalDelete = this.sql
}
this.incrementalConfig.tableId = this.table.id
post('/dataset/table/save/incrementalConfig', this.incrementalConfig).then(response => {
this.$message({
message: this.$t('dataset.save_success'),
type: 'success',
showClose: true
})
})
},
saveTask(task) {
task.startTime = new Date(task.startTime).getTime()
task.endTime = new Date(task.endTime).getTime()
@ -289,18 +431,6 @@ export default {
this.resetTaskForm()
this.listTask()
})
// this.$refs['taskForm'].validate((valid) => {
// if (valid) {
//
// } else {
// this.$message({
// message: this.$t('dataset.input_content'),
// type: 'error',
// showClose: true
// })
// return false
// }
// })
},
deleteTask(task) {
this.$confirm(this.$t('dataset.confirm_delete'), this.$t('dataset.tips'), {
@ -325,7 +455,7 @@ export default {
this.resetTaskForm()
},
onRateChange() {
if (this.taskForm.rate === '0') {
if (this.taskForm.rate === 'SIMPLE') {
this.taskForm.end = '0'
this.taskForm.endTime = ''
this.taskForm.cron = ''
@ -346,13 +476,25 @@ export default {
resetTaskForm() {
this.taskForm = {
name: '',
type: '0',
type: 'all_scope',
startTime: '',
rate: '0',
rate: 'SIMPLE',
endTime: '',
end: '0'
}
},
onCmReady(cm) {
this.codemirror.setSize('-webkit-fill-available', 'auto')
},
onCmFocus(cm) {
// console.log('the editor is focus!', cm)
},
onCmCodeChange(newCode) {
console.log(newCode)
this.sql = newCode
this.$emit('codeChange', this.sql)
}
}
}
</script>
@ -369,4 +511,13 @@ export default {
.el-form-item {
margin-bottom: 10px;
}
.codemirror {
height: 160px;
overflow-y: auto;
}
.codemirror >>> .CodeMirror-scroll {
height: 160px;
overflow-y: auto;
}
</style>

11
frontend/src/views/system/datasource/index.vue
View File

@ -11,7 +11,7 @@
@search="search"
>
<template #buttons>
<fu-table-button icon="el-icon-circle-plus-outline" :label="$t('datasource.create')" @click="create" />
<fu-table-button v-permission="['datasource:add']" icon="el-icon-circle-plus-outline" :label="$t('datasource.create')" @click="create" />
</template>
<!-- <el-table-column type="selection" fix /> -->
@ -62,7 +62,7 @@
<el-form-item v-show="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.data_base')" prop="configuration.dataBase" :rules="{required: true, message: $t('datasource.please_input_data_base'), trigger: 'blur'}">
<el-input v-model="form.configuration.dataBase" autocomplete="off" />
</el-form-item>
<el-form-item v-show="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.user_name')" prop="configuration.username">
<el-form-item v-show="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.user_name')" prop="configuration.username" :rules="{required: true, message: $t('datasource.please_input_user_name'), trigger: 'blur'}">
<el-input v-model="form.configuration.username" autocomplete="off" />
</el-form-item>
<el-form-item v-show="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.password')" prop="configuration.password" :rules="{required: true, message: $t('datasource.please_input_password'), trigger: 'change'}">
@ -79,7 +79,8 @@
<div slot="footer" class="dialog-footer">
<el-button type="text" @click="dialogVisible = false">{{ $t('commons.cancel') }}</el-button>
<el-button type="primary" @click="saveDatasource('createDatasource')">确认</el-button>
<el-button type="primary" @click="validaDatasource('createDatasource')">{{ $t('commons.validate') }}</el-button>
<el-button type="primary" @click="saveDatasource('createDatasource')">{{ $t('commons.confirm') }}</el-button>
</div>
</el-dialog>
@ -198,13 +199,12 @@ export default {
this.$success(this.$t('commons.save_success'))
this.search()
this.dialogVisible = false
})
});
} else {
return false
}
})
},
validaDatasource(datasourceForm) {
this.$refs[datasourceForm].validate(valid => {
if (valid) {
@ -230,7 +230,6 @@ export default {
const result = {}
if (condition && condition.quick) {
for (const [key, value] of Object.entries(condition)) {
// console.log(`${key}`)
if (`${key}` === 'quick') {
const v_new = Object.assign({}, value)
v_new['field'] = 'name'