diff --git a/backend/src/main/java/io/dataease/base/domain/DeEngine.java b/backend/src/main/java/io/dataease/base/domain/DeEngine.java new file mode 100644 index 0000000000..0849151d75 --- /dev/null +++ b/backend/src/main/java/io/dataease/base/domain/DeEngine.java @@ -0,0 +1,27 @@ +package io.dataease.base.domain; + +import java.io.Serializable; +import lombok.Data; + +@Data +public class DeEngine implements Serializable { + private String id; + + private String name; + + private String desc; + + private String type; + + private Long createTime; + + private Long updateTime; + + private String createBy; + + private String status; + + private String configuration; + + private static final long serialVersionUID = 1L; +} \ No newline at end of file diff --git a/backend/src/main/java/io/dataease/base/domain/DeEngineExample.java b/backend/src/main/java/io/dataease/base/domain/DeEngineExample.java new file mode 100644 index 0000000000..243c047ddc --- /dev/null +++ b/backend/src/main/java/io/dataease/base/domain/DeEngineExample.java @@ -0,0 +1,740 @@ +package io.dataease.base.domain; + +import java.util.ArrayList; +import java.util.List; + +public class DeEngineExample { + protected String orderByClause; + + protected boolean distinct; + + protected List oredCriteria; + + public DeEngineExample() { + oredCriteria = new ArrayList(); + } + + public void setOrderByClause(String orderByClause) { + this.orderByClause = orderByClause; + } + + public String getOrderByClause() { + return orderByClause; + } + + public void setDistinct(boolean distinct) { + this.distinct = distinct; + } + + public boolean isDistinct() { + return distinct; + } + + public List getOredCriteria() { + return oredCriteria; + } + + public void or(Criteria criteria) { + oredCriteria.add(criteria); + } + + public Criteria or() { + Criteria criteria = createCriteriaInternal(); + oredCriteria.add(criteria); + return criteria; + } + + public Criteria createCriteria() { + Criteria criteria = createCriteriaInternal(); + if (oredCriteria.size() == 0) { + oredCriteria.add(criteria); + } + return criteria; + } + + protected Criteria createCriteriaInternal() { + Criteria criteria = new Criteria(); + return criteria; + } + + public void clear() { + oredCriteria.clear(); + orderByClause = null; + distinct = false; + } + + protected abstract static class GeneratedCriteria { + protected List criteria; + + protected GeneratedCriteria() { + super(); + criteria = new ArrayList(); + } + + public boolean isValid() { + return criteria.size() > 0; + } + + public List getAllCriteria() { + return criteria; + } + + public List getCriteria() { + return criteria; + } + + protected void addCriterion(String condition) { + if (condition == null) { + throw new RuntimeException("Value for condition cannot be null"); + } + criteria.add(new Criterion(condition)); + } + + protected void addCriterion(String condition, Object value, String property) { + if (value == null) { + throw new RuntimeException("Value for " + property + " cannot be null"); + } + criteria.add(new Criterion(condition, value)); + } + + protected void addCriterion(String condition, Object value1, Object value2, String property) { + if (value1 == null || value2 == null) { + throw new RuntimeException("Between values for " + property + " cannot be null"); + } + criteria.add(new Criterion(condition, value1, value2)); + } + + public Criteria andIdIsNull() { + addCriterion("id is null"); + return (Criteria) this; + } + + public Criteria andIdIsNotNull() { + addCriterion("id is not null"); + return (Criteria) this; + } + + public Criteria andIdEqualTo(String value) { + addCriterion("id =", value, "id"); + return (Criteria) this; + } + + public Criteria andIdNotEqualTo(String value) { + addCriterion("id <>", value, "id"); + return (Criteria) this; + } + + public Criteria andIdGreaterThan(String value) { + addCriterion("id >", value, "id"); + return (Criteria) this; + } + + public Criteria andIdGreaterThanOrEqualTo(String value) { + addCriterion("id >=", value, "id"); + return (Criteria) this; + } + + public Criteria andIdLessThan(String value) { + addCriterion("id <", value, "id"); + return (Criteria) this; + } + + public Criteria andIdLessThanOrEqualTo(String value) { + addCriterion("id <=", value, "id"); + return (Criteria) this; + } + + public Criteria andIdLike(String value) { + addCriterion("id like", value, "id"); + return (Criteria) this; + } + + public Criteria andIdNotLike(String value) { + addCriterion("id not like", value, "id"); + return (Criteria) this; + } + + public Criteria andIdIn(List values) { + addCriterion("id in", values, "id"); + return (Criteria) this; + } + + public Criteria andIdNotIn(List values) { + addCriterion("id not in", values, "id"); + return (Criteria) this; + } + + public Criteria andIdBetween(String value1, String value2) { + addCriterion("id between", value1, value2, "id"); + return (Criteria) this; + } + + public Criteria andIdNotBetween(String value1, String value2) { + addCriterion("id not between", value1, value2, "id"); + return (Criteria) this; + } + + public Criteria andNameIsNull() { + addCriterion("`name` is null"); + return (Criteria) this; + } + + public Criteria andNameIsNotNull() { + addCriterion("`name` is not null"); + return (Criteria) this; + } + + public Criteria andNameEqualTo(String value) { + addCriterion("`name` =", value, "name"); + return (Criteria) this; + } + + public Criteria andNameNotEqualTo(String value) { + addCriterion("`name` <>", value, "name"); + return (Criteria) this; + } + + public Criteria andNameGreaterThan(String value) { + addCriterion("`name` >", value, "name"); + return (Criteria) this; + } + + public Criteria andNameGreaterThanOrEqualTo(String value) { + addCriterion("`name` >=", value, "name"); + return (Criteria) this; + } + + public Criteria andNameLessThan(String value) { + addCriterion("`name` <", value, "name"); + return (Criteria) this; + } + + public Criteria andNameLessThanOrEqualTo(String value) { + addCriterion("`name` <=", value, "name"); + return (Criteria) this; + } + + public Criteria andNameLike(String value) { + addCriterion("`name` like", value, "name"); + return (Criteria) this; + } + + public Criteria andNameNotLike(String value) { + addCriterion("`name` not like", value, "name"); + return (Criteria) this; + } + + public Criteria andNameIn(List values) { + addCriterion("`name` in", values, "name"); + return (Criteria) this; + } + + public Criteria andNameNotIn(List values) { + addCriterion("`name` not in", values, "name"); + return (Criteria) this; + } + + public Criteria andNameBetween(String value1, String value2) { + addCriterion("`name` between", value1, value2, "name"); + return (Criteria) this; + } + + public Criteria andNameNotBetween(String value1, String value2) { + addCriterion("`name` not between", value1, value2, "name"); + return (Criteria) this; + } + + public Criteria andDescIsNull() { + addCriterion("`desc` is null"); + return (Criteria) this; + } + + public Criteria andDescIsNotNull() { + addCriterion("`desc` is not null"); + return (Criteria) this; + } + + public Criteria andDescEqualTo(String value) { + addCriterion("`desc` =", value, "desc"); + return (Criteria) this; + } + + public Criteria andDescNotEqualTo(String value) { + addCriterion("`desc` <>", value, "desc"); + return (Criteria) this; + } + + public Criteria andDescGreaterThan(String value) { + addCriterion("`desc` >", value, "desc"); + return (Criteria) this; + } + + public Criteria andDescGreaterThanOrEqualTo(String value) { + addCriterion("`desc` >=", value, "desc"); + return (Criteria) this; + } + + public Criteria andDescLessThan(String value) { + addCriterion("`desc` <", value, "desc"); + return (Criteria) this; + } + + public Criteria andDescLessThanOrEqualTo(String value) { + addCriterion("`desc` <=", value, "desc"); + return (Criteria) this; + } + + public Criteria andDescLike(String value) { + addCriterion("`desc` like", value, "desc"); + return (Criteria) this; + } + + public Criteria andDescNotLike(String value) { + addCriterion("`desc` not like", value, "desc"); + return (Criteria) this; + } + + public Criteria andDescIn(List values) { + addCriterion("`desc` in", values, "desc"); + return (Criteria) this; + } + + public Criteria andDescNotIn(List values) { + addCriterion("`desc` not in", values, "desc"); + return (Criteria) this; + } + + public Criteria andDescBetween(String value1, String value2) { + addCriterion("`desc` between", value1, value2, "desc"); + return (Criteria) this; + } + + public Criteria andDescNotBetween(String value1, String value2) { + addCriterion("`desc` not between", value1, value2, "desc"); + return (Criteria) this; + } + + public Criteria andTypeIsNull() { + addCriterion("`type` is null"); + return (Criteria) this; + } + + public Criteria andTypeIsNotNull() { + addCriterion("`type` is not null"); + return (Criteria) this; + } + + public Criteria andTypeEqualTo(String value) { + addCriterion("`type` =", value, "type"); + return (Criteria) this; + } + + public Criteria andTypeNotEqualTo(String value) { + addCriterion("`type` <>", value, "type"); + return (Criteria) this; + } + + public Criteria andTypeGreaterThan(String value) { + addCriterion("`type` >", value, "type"); + return (Criteria) this; + } + + public Criteria andTypeGreaterThanOrEqualTo(String value) { + addCriterion("`type` >=", value, "type"); + return (Criteria) this; + } + + public Criteria andTypeLessThan(String value) { + addCriterion("`type` <", value, "type"); + return (Criteria) this; + } + + public Criteria andTypeLessThanOrEqualTo(String value) { + addCriterion("`type` <=", value, "type"); + return (Criteria) this; + } + + public Criteria andTypeLike(String value) { + addCriterion("`type` like", value, "type"); + return (Criteria) this; + } + + public Criteria andTypeNotLike(String value) { + addCriterion("`type` not like", value, "type"); + return (Criteria) this; + } + + public Criteria andTypeIn(List values) { + addCriterion("`type` in", values, "type"); + return (Criteria) this; + } + + public Criteria andTypeNotIn(List values) { + addCriterion("`type` not in", values, "type"); + return (Criteria) this; + } + + public Criteria andTypeBetween(String value1, String value2) { + addCriterion("`type` between", value1, value2, "type"); + return (Criteria) this; + } + + public Criteria andTypeNotBetween(String value1, String value2) { + addCriterion("`type` not between", value1, value2, "type"); + return (Criteria) this; + } + + public Criteria andCreateTimeIsNull() { + addCriterion("create_time is null"); + return (Criteria) this; + } + + public Criteria andCreateTimeIsNotNull() { + addCriterion("create_time is not null"); + return (Criteria) this; + } + + public Criteria andCreateTimeEqualTo(Long value) { + addCriterion("create_time =", value, "createTime"); + return (Criteria) this; + } + + public Criteria andCreateTimeNotEqualTo(Long value) { + addCriterion("create_time <>", value, "createTime"); + return (Criteria) this; + } + + public Criteria andCreateTimeGreaterThan(Long value) { + addCriterion("create_time >", value, "createTime"); + return (Criteria) this; + } + + public Criteria andCreateTimeGreaterThanOrEqualTo(Long value) { + addCriterion("create_time >=", value, "createTime"); + return (Criteria) this; + } + + public Criteria andCreateTimeLessThan(Long value) { + addCriterion("create_time <", value, "createTime"); + return (Criteria) this; + } + + public Criteria andCreateTimeLessThanOrEqualTo(Long value) { + addCriterion("create_time <=", value, "createTime"); + return (Criteria) this; + } + + public Criteria andCreateTimeIn(List values) { + addCriterion("create_time in", values, "createTime"); + return (Criteria) this; + } + + public Criteria andCreateTimeNotIn(List values) { + addCriterion("create_time not in", values, "createTime"); + return (Criteria) this; + } + + public Criteria andCreateTimeBetween(Long value1, Long value2) { + addCriterion("create_time between", value1, value2, "createTime"); + return (Criteria) this; + } + + public Criteria andCreateTimeNotBetween(Long value1, Long value2) { + addCriterion("create_time not between", value1, value2, "createTime"); + return (Criteria) this; + } + + public Criteria andUpdateTimeIsNull() { + addCriterion("update_time is null"); + return (Criteria) this; + } + + public Criteria andUpdateTimeIsNotNull() { + addCriterion("update_time is not null"); + return (Criteria) this; + } + + public Criteria andUpdateTimeEqualTo(Long value) { + addCriterion("update_time =", value, "updateTime"); + return (Criteria) this; + } + + public Criteria andUpdateTimeNotEqualTo(Long value) { + addCriterion("update_time <>", value, "updateTime"); + return (Criteria) this; + } + + public Criteria andUpdateTimeGreaterThan(Long value) { + addCriterion("update_time >", value, "updateTime"); + return (Criteria) this; + } + + public Criteria andUpdateTimeGreaterThanOrEqualTo(Long value) { + addCriterion("update_time >=", value, "updateTime"); + return (Criteria) this; + } + + public Criteria andUpdateTimeLessThan(Long value) { + addCriterion("update_time <", value, "updateTime"); + return (Criteria) this; + } + + public Criteria andUpdateTimeLessThanOrEqualTo(Long value) { + addCriterion("update_time <=", value, "updateTime"); + return (Criteria) this; + } + + public Criteria andUpdateTimeIn(List values) { + addCriterion("update_time in", values, "updateTime"); + return (Criteria) this; + } + + public Criteria andUpdateTimeNotIn(List values) { + addCriterion("update_time not in", values, "updateTime"); + return (Criteria) this; + } + + public Criteria andUpdateTimeBetween(Long value1, Long value2) { + addCriterion("update_time between", value1, value2, "updateTime"); + return (Criteria) this; + } + + public Criteria andUpdateTimeNotBetween(Long value1, Long value2) { + addCriterion("update_time not between", value1, value2, "updateTime"); + return (Criteria) this; + } + + public Criteria andCreateByIsNull() { + addCriterion("create_by is null"); + return (Criteria) this; + } + + public Criteria andCreateByIsNotNull() { + addCriterion("create_by is not null"); + return (Criteria) this; + } + + public Criteria andCreateByEqualTo(String value) { + addCriterion("create_by =", value, "createBy"); + return (Criteria) this; + } + + public Criteria andCreateByNotEqualTo(String value) { + addCriterion("create_by <>", value, "createBy"); + return (Criteria) this; + } + + public Criteria andCreateByGreaterThan(String value) { + addCriterion("create_by >", value, "createBy"); + return (Criteria) this; + } + + public Criteria andCreateByGreaterThanOrEqualTo(String value) { + addCriterion("create_by >=", value, "createBy"); + return (Criteria) this; + } + + public Criteria andCreateByLessThan(String value) { + addCriterion("create_by <", value, "createBy"); + return (Criteria) this; + } + + public Criteria andCreateByLessThanOrEqualTo(String value) { + addCriterion("create_by <=", value, "createBy"); + return (Criteria) this; + } + + public Criteria andCreateByLike(String value) { + addCriterion("create_by like", value, "createBy"); + return (Criteria) this; + } + + public Criteria andCreateByNotLike(String value) { + addCriterion("create_by not like", value, "createBy"); + return (Criteria) this; + } + + public Criteria andCreateByIn(List values) { + addCriterion("create_by in", values, "createBy"); + return (Criteria) this; + } + + public Criteria andCreateByNotIn(List values) { + addCriterion("create_by not in", values, "createBy"); + return (Criteria) this; + } + + public Criteria andCreateByBetween(String value1, String value2) { + addCriterion("create_by between", value1, value2, "createBy"); + return (Criteria) this; + } + + public Criteria andCreateByNotBetween(String value1, String value2) { + addCriterion("create_by not between", value1, value2, "createBy"); + return (Criteria) this; + } + + public Criteria andStatusIsNull() { + addCriterion("`status` is null"); + return (Criteria) this; + } + + public Criteria andStatusIsNotNull() { + addCriterion("`status` is not null"); + return (Criteria) this; + } + + public Criteria andStatusEqualTo(String value) { + addCriterion("`status` =", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotEqualTo(String value) { + addCriterion("`status` <>", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusGreaterThan(String value) { + addCriterion("`status` >", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusGreaterThanOrEqualTo(String value) { + addCriterion("`status` >=", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusLessThan(String value) { + addCriterion("`status` <", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusLessThanOrEqualTo(String value) { + addCriterion("`status` <=", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusLike(String value) { + addCriterion("`status` like", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotLike(String value) { + addCriterion("`status` not like", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusIn(List values) { + addCriterion("`status` in", values, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotIn(List values) { + addCriterion("`status` not in", values, "status"); + return (Criteria) this; + } + + public Criteria andStatusBetween(String value1, String value2) { + addCriterion("`status` between", value1, value2, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotBetween(String value1, String value2) { + addCriterion("`status` not between", value1, value2, "status"); + return (Criteria) this; + } + } + + public static class Criteria extends GeneratedCriteria { + + protected Criteria() { + super(); + } + } + + public static class Criterion { + private String condition; + + private Object value; + + private Object secondValue; + + private boolean noValue; + + private boolean singleValue; + + private boolean betweenValue; + + private boolean listValue; + + private String typeHandler; + + public String getCondition() { + return condition; + } + + public Object getValue() { + return value; + } + + public Object getSecondValue() { + return secondValue; + } + + public boolean isNoValue() { + return noValue; + } + + public boolean isSingleValue() { + return singleValue; + } + + public boolean isBetweenValue() { + return betweenValue; + } + + public boolean isListValue() { + return listValue; + } + + public String getTypeHandler() { + return typeHandler; + } + + protected Criterion(String condition) { + super(); + this.condition = condition; + this.typeHandler = null; + this.noValue = true; + } + + protected Criterion(String condition, Object value, String typeHandler) { + super(); + this.condition = condition; + this.value = value; + this.typeHandler = typeHandler; + if (value instanceof List) { + this.listValue = true; + } else { + this.singleValue = true; + } + } + + protected Criterion(String condition, Object value) { + this(condition, value, null); + } + + protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { + super(); + this.condition = condition; + this.value = value; + this.secondValue = secondValue; + this.typeHandler = typeHandler; + this.betweenValue = true; + } + + protected Criterion(String condition, Object value, Object secondValue) { + this(condition, value, secondValue, null); + } + } +} \ No newline at end of file diff --git a/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.java b/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.java new file mode 100644 index 0000000000..2bcf37b094 --- /dev/null +++ b/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.java @@ -0,0 +1,36 @@ +package io.dataease.base.mapper; + +import io.dataease.base.domain.DeEngine; +import io.dataease.base.domain.DeEngineExample; +import java.util.List; +import org.apache.ibatis.annotations.Param; + +public interface DeEngineMapper { + long countByExample(DeEngineExample example); + + int deleteByExample(DeEngineExample example); + + int deleteByPrimaryKey(String id); + + int insert(DeEngine record); + + int insertSelective(DeEngine record); + + List selectByExampleWithBLOBs(DeEngineExample example); + + List selectByExample(DeEngineExample example); + + DeEngine selectByPrimaryKey(String id); + + int updateByExampleSelective(@Param("record") DeEngine record, @Param("example") DeEngineExample example); + + int updateByExampleWithBLOBs(@Param("record") DeEngine record, @Param("example") DeEngineExample example); + + int updateByExample(@Param("record") DeEngine record, @Param("example") DeEngineExample example); + + int updateByPrimaryKeySelective(DeEngine record); + + int updateByPrimaryKeyWithBLOBs(DeEngine record); + + int updateByPrimaryKey(DeEngine record); +} \ No newline at end of file diff --git a/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.xml b/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.xml new file mode 100644 index 0000000000..dd37c69f8a --- /dev/null +++ b/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.xml @@ -0,0 +1,323 @@ + + + + + + + + + + + + + + + + + + + + + + + + + and ${criterion.condition} + + + and ${criterion.condition} #{criterion.value} + + + and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} + + + and ${criterion.condition} + + #{listItem} + + + + + + + + + + + + + + + + + + and ${criterion.condition} + + + and ${criterion.condition} #{criterion.value} + + + and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} + + + and ${criterion.condition} + + #{listItem} + + + + + + + + + + + id, `name`, `desc`, `type`, create_time, update_time, create_by, `status` + + + configuration + + + + + + delete from de_engine + where id = #{id,jdbcType=VARCHAR} + + + delete from de_engine + + + + + + insert into de_engine (id, `name`, `desc`, + `type`, create_time, update_time, + create_by, `status`, configuration + ) + values (#{id,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{desc,jdbcType=VARCHAR}, + #{type,jdbcType=VARCHAR}, #{createTime,jdbcType=BIGINT}, #{updateTime,jdbcType=BIGINT}, + #{createBy,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{configuration,jdbcType=LONGVARCHAR} + ) + + + insert into de_engine + + + id, + + + `name`, + + + `desc`, + + + `type`, + + + create_time, + + + update_time, + + + create_by, + + + `status`, + + + configuration, + + + + + #{id,jdbcType=VARCHAR}, + + + #{name,jdbcType=VARCHAR}, + + + #{desc,jdbcType=VARCHAR}, + + + #{type,jdbcType=VARCHAR}, + + + #{createTime,jdbcType=BIGINT}, + + + #{updateTime,jdbcType=BIGINT}, + + + #{createBy,jdbcType=VARCHAR}, + + + #{status,jdbcType=VARCHAR}, + + + #{configuration,jdbcType=LONGVARCHAR}, + + + + + + update de_engine + + + id = #{record.id,jdbcType=VARCHAR}, + + + `name` = #{record.name,jdbcType=VARCHAR}, + + + `desc` = #{record.desc,jdbcType=VARCHAR}, + + + `type` = #{record.type,jdbcType=VARCHAR}, + + + create_time = #{record.createTime,jdbcType=BIGINT}, + + + update_time = #{record.updateTime,jdbcType=BIGINT}, + + + create_by = #{record.createBy,jdbcType=VARCHAR}, + + + `status` = #{record.status,jdbcType=VARCHAR}, + + + configuration = #{record.configuration,jdbcType=LONGVARCHAR}, + + + + + + + + update de_engine + set id = #{record.id,jdbcType=VARCHAR}, + `name` = #{record.name,jdbcType=VARCHAR}, + `desc` = #{record.desc,jdbcType=VARCHAR}, + `type` = #{record.type,jdbcType=VARCHAR}, + create_time = #{record.createTime,jdbcType=BIGINT}, + update_time = #{record.updateTime,jdbcType=BIGINT}, + create_by = #{record.createBy,jdbcType=VARCHAR}, + `status` = #{record.status,jdbcType=VARCHAR}, + configuration = #{record.configuration,jdbcType=LONGVARCHAR} + + + + + + update de_engine + set id = #{record.id,jdbcType=VARCHAR}, + `name` = #{record.name,jdbcType=VARCHAR}, + `desc` = #{record.desc,jdbcType=VARCHAR}, + `type` = #{record.type,jdbcType=VARCHAR}, + create_time = #{record.createTime,jdbcType=BIGINT}, + update_time = #{record.updateTime,jdbcType=BIGINT}, + create_by = #{record.createBy,jdbcType=VARCHAR}, + `status` = #{record.status,jdbcType=VARCHAR} + + + + + + update de_engine + + + `name` = #{name,jdbcType=VARCHAR}, + + + `desc` = #{desc,jdbcType=VARCHAR}, + + + `type` = #{type,jdbcType=VARCHAR}, + + + create_time = #{createTime,jdbcType=BIGINT}, + + + update_time = #{updateTime,jdbcType=BIGINT}, + + + create_by = #{createBy,jdbcType=VARCHAR}, + + + `status` = #{status,jdbcType=VARCHAR}, + + + configuration = #{configuration,jdbcType=LONGVARCHAR}, + + + where id = #{id,jdbcType=VARCHAR} + + + update de_engine + set `name` = #{name,jdbcType=VARCHAR}, + `desc` = #{desc,jdbcType=VARCHAR}, + `type` = #{type,jdbcType=VARCHAR}, + create_time = #{createTime,jdbcType=BIGINT}, + update_time = #{updateTime,jdbcType=BIGINT}, + create_by = #{createBy,jdbcType=VARCHAR}, + `status` = #{status,jdbcType=VARCHAR}, + configuration = #{configuration,jdbcType=LONGVARCHAR} + where id = #{id,jdbcType=VARCHAR} + + + update de_engine + set `name` = #{name,jdbcType=VARCHAR}, + `desc` = #{desc,jdbcType=VARCHAR}, + `type` = #{type,jdbcType=VARCHAR}, + create_time = #{createTime,jdbcType=BIGINT}, + update_time = #{updateTime,jdbcType=BIGINT}, + create_by = #{createBy,jdbcType=VARCHAR}, + `status` = #{status,jdbcType=VARCHAR} + where id = #{id,jdbcType=VARCHAR} + + \ No newline at end of file diff --git a/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java b/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java index 1250a90a08..a28cbba4a4 100644 --- a/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java +++ b/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java @@ -8,14 +8,15 @@ public enum DatasourceTypes { ds_doris("ds_doris", "ds_doris", "com.mysql.jdbc.Driver", "`", "`", "'", "'"), pg("pg", "pg", "org.postgresql.Driver", "\"", "\"", "\"", "\""), sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""), - de_doris("de_doris", "de_doris", "com.mysql.jdbc.Driver", "`", "`", "", ""), oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""), mongo("mongo", "mongodb", "com.mongodb.jdbc.MongoDriver", "`", "`", "\"", "\""), ck("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"), db2("db2", "db2", "com.ibm.db2.jcc.DB2Driver", "\"", "\"", "\"", "\""), es("es", "es", "", "\"", "\"", "\"", "\""), redshift("redshift", "redshift", "org.postgresql.Driver", "\"", "\"", "\"", "\""), - api("api", "api", "", "\"", "\"", "\"", "\""); + api("api", "api", "", "\"", "\"", "\"", "\""), + engine_doris("engine_doris", "engine_doris", "com.mysql.jdbc.Driver", "`", "`", "", ""), + engine_mysql("mysql", "mysql", "com.mysql.jdbc.Driver", "`", "`", "'", "'"); private String feature; diff --git a/backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java b/backend/src/main/java/io/dataease/commons/utils/TableUtils.java similarity index 54% rename from backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java rename to backend/src/main/java/io/dataease/commons/utils/TableUtils.java index 154b5aadb4..b2bdfafb2b 100644 --- a/backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java +++ b/backend/src/main/java/io/dataease/commons/utils/TableUtils.java @@ -1,28 +1,28 @@ package io.dataease.commons.utils; -public class DorisTableUtils { +public class TableUtils { - public static String dorisName(String datasetId) { + public static String tableName(String datasetId) { return "ds_" + datasetId.replace("-", "_"); } - public static String dorisTmpName(String dorisName) { + public static String tmpName(String dorisName) { return "tmp_" + dorisName; } - public static String dorisDeleteName(String dorisName) { + public static String deleteName(String dorisName) { return "delete_" + dorisName; } - public static String dorisAddName(String dorisName) { + public static String addName(String dorisName) { return "add_" + dorisName; } - public static String dorisFieldName(String dorisName) { + public static String fieldName(String dorisName) { return "f_" + Md5Utils.md5(dorisName); } - public static String dorisFieldNameShort(String dorisName) { + public static String fieldNameShort(String dorisName) { return "f_" + Md5Utils.md5(dorisName).substring(8, 24); } diff --git a/backend/src/main/java/io/dataease/config/CommonConfig.java b/backend/src/main/java/io/dataease/config/CommonConfig.java index 0792dd0f46..77979c0308 100644 --- a/backend/src/main/java/io/dataease/config/CommonConfig.java +++ b/backend/src/main/java/io/dataease/config/CommonConfig.java @@ -23,28 +23,6 @@ public class CommonConfig { private Environment env; // 保存了配置文件的信息 private static String root_path = "/opt/dataease/data/kettle/"; - @Bean(name = "DorisDatasource") - @ConditionalOnMissingBean - public Datasource configuration() { - JSONObject jsonObject = new JSONObject(); - jsonObject.put("dataSourceType", "jdbc"); - jsonObject.put("dataBase", env.getProperty("doris.db", "doris")); - jsonObject.put("username", env.getProperty("doris.user", "root")); - jsonObject.put("password", env.getProperty("doris.password", "dataease")); - jsonObject.put("host", env.getProperty("doris.host", "doris")); - jsonObject.put("port", env.getProperty("doris.port", "9030")); - jsonObject.put("httpPort", env.getProperty("doris.httpPort", "8030")); - - Datasource datasource = new Datasource(); - datasource.setId("doris"); - datasource.setName("doris"); - datasource.setDesc("doris"); - datasource.setType("de_doris"); - datasource.setConfiguration(jsonObject.toJSONString()); - return datasource; - } - - @Bean @ConditionalOnMissingBean public KettleFileRepository kettleFileRepository() throws Exception { diff --git a/backend/src/main/java/io/dataease/controller/engine/EngineController.java b/backend/src/main/java/io/dataease/controller/engine/EngineController.java new file mode 100644 index 0000000000..c032534158 --- /dev/null +++ b/backend/src/main/java/io/dataease/controller/engine/EngineController.java @@ -0,0 +1,45 @@ +package io.dataease.controller.engine; + +import io.dataease.base.domain.DeEngine; +import io.dataease.controller.ResultHolder; +import io.dataease.dto.DatasourceDTO; +import io.dataease.service.engine.EngineService; +import org.springframework.web.bind.annotation.*; +import springfox.documentation.annotations.ApiIgnore; + +import javax.annotation.Resource; + +@ApiIgnore +@RequestMapping("engine") +@RestController +public class EngineController { + + @Resource + private EngineService engineService; + + @ApiIgnore + @GetMapping("/mode") + public String runMode() throws Exception{ + return engineService.mode(); + } + + @ApiIgnore + @GetMapping("/info") + public DeEngine info() throws Exception{ + return engineService.info(); + } + + @ApiIgnore + @PostMapping("/validate") + public ResultHolder validate(@RequestBody DatasourceDTO datasource) throws Exception { + return engineService.validate(datasource); + } + + + @ApiIgnore + @PostMapping("/save") + public ResultHolder save(@RequestBody DeEngine engine) throws Exception { + return engineService.save(engine); + } + +} diff --git a/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java b/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java index 453972e4dd..8918c488b8 100644 --- a/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java +++ b/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java @@ -20,5 +20,8 @@ public class DataSourceInitStartListener implements ApplicationListener datasetTableFields); + + public abstract String insertSql(String name, List dataList, int page, int pageNumber); +} diff --git a/backend/src/main/java/io/dataease/provider/DDLProviderImpl.java b/backend/src/main/java/io/dataease/provider/DDLProviderImpl.java new file mode 100644 index 0000000000..ada0dd0294 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/DDLProviderImpl.java @@ -0,0 +1,50 @@ +package io.dataease.provider; + +import io.dataease.base.domain.DatasetTableField; +import io.dataease.commons.utils.Md5Utils; + +import java.util.Arrays; +import java.util.List; + +public class DDLProviderImpl extends DDLProvider { + @Override + public String createView(String name, String viewSQL) { + return null; + } + + @Override + public String dropTable(String name) { + return null; + } + + @Override + public String dropView(String name) { + return null; + } + + @Override + public String replaceTable(String name) { + return null; + } + + @Override + public String createTableSql(String name, List datasetTableFields) { + return null; + } + + @Override + public String insertSql(String name, List dataList, int page, int pageNumber) { + String insertSql = "INSERT INTO TABLE_NAME VALUES ".replace("TABLE_NAME", name); + StringBuffer values = new StringBuffer(); + + Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size(); + for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) { + values.append("(").append(Md5Utils.md5(String.join(",", Arrays.asList(strings)))) + .append("," ).append(String.join(",", Arrays.asList(strings))) + .append("),"); + } + return insertSql + values.substring(0, values.length() - 1); + } + + +} diff --git a/backend/src/main/java/io/dataease/provider/ProviderFactory.java b/backend/src/main/java/io/dataease/provider/ProviderFactory.java index b901fbb02b..3dc13a582b 100644 --- a/backend/src/main/java/io/dataease/provider/ProviderFactory.java +++ b/backend/src/main/java/io/dataease/provider/ProviderFactory.java @@ -2,8 +2,6 @@ package io.dataease.provider; import io.dataease.commons.constants.DatasourceTypes; import io.dataease.provider.datasource.DatasourceProvider; -import io.dataease.provider.query.DDLProvider; -import io.dataease.provider.query.QueryProvider; import io.dataease.provider.query.api.ApiProvider; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; @@ -40,8 +38,6 @@ public class ProviderFactory implements ApplicationContextAware { case mariadb: case ds_doris: return context.getBean("mysqlQuery", QueryProvider.class); - case de_doris: - return context.getBean("dorisQuery", QueryProvider.class); case sqlServer: return context.getBean("sqlserverQuery", QueryProvider.class); case pg: @@ -62,6 +58,10 @@ public class ProviderFactory implements ApplicationContextAware { return context.getBean("db2Query", QueryProvider.class); case api: return context.getBean("apiQuery", ApiProvider.class); + case engine_doris: + return context.getBean("dorisEngineQuery", QueryProvider.class); + case engine_mysql: + return context.getBean("mysqlEngineQuery", QueryProvider.class); default: return context.getBean("mysqlQuery", QueryProvider.class); } @@ -70,10 +70,12 @@ public class ProviderFactory implements ApplicationContextAware { public static DDLProvider getDDLProvider(String type) { DatasourceTypes datasourceType = DatasourceTypes.valueOf(type); switch (datasourceType) { - case de_doris: - return context.getBean("dorisDDL", DDLProvider.class); + case engine_doris: + return context.getBean("dorisEngineDDL", DDLProvider.class); + case engine_mysql: + return context.getBean("mysqlEngineDDL", DDLProvider.class); default: - return context.getBean("dorisDDL", DDLProvider.class); + return context.getBean("dorisEngineDDL", DDLProvider.class); } } diff --git a/backend/src/main/java/io/dataease/provider/query/QueryProvider.java b/backend/src/main/java/io/dataease/provider/QueryProvider.java similarity index 99% rename from backend/src/main/java/io/dataease/provider/query/QueryProvider.java rename to backend/src/main/java/io/dataease/provider/QueryProvider.java index 8716317f0b..c7eca21213 100644 --- a/backend/src/main/java/io/dataease/provider/query/QueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/QueryProvider.java @@ -1,4 +1,4 @@ -package io.dataease.provider.query; +package io.dataease.provider; import com.google.gson.Gson; import io.dataease.base.domain.ChartViewWithBLOBs; diff --git a/backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java b/backend/src/main/java/io/dataease/provider/QueryProviderImpl.java similarity index 99% rename from backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java rename to backend/src/main/java/io/dataease/provider/QueryProviderImpl.java index 548713b458..7d8083e2ee 100644 --- a/backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java +++ b/backend/src/main/java/io/dataease/provider/QueryProviderImpl.java @@ -1,4 +1,4 @@ -package io.dataease.provider.query; +package io.dataease.provider; import io.dataease.base.domain.ChartViewWithBLOBs; import io.dataease.base.domain.DatasetTableField; diff --git a/backend/src/main/java/io/dataease/provider/query/SQLConstants.java b/backend/src/main/java/io/dataease/provider/SQLConstants.java similarity index 96% rename from backend/src/main/java/io/dataease/provider/query/SQLConstants.java rename to backend/src/main/java/io/dataease/provider/SQLConstants.java index e7174b5211..3fbe98f655 100644 --- a/backend/src/main/java/io/dataease/provider/query/SQLConstants.java +++ b/backend/src/main/java/io/dataease/provider/SQLConstants.java @@ -1,4 +1,4 @@ -package io.dataease.provider.query; +package io.dataease.provider; import java.util.ArrayList; import java.util.List; diff --git a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java index 1c8cd2a827..d135676871 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java @@ -11,7 +11,7 @@ import io.dataease.dto.datasource.*; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; import io.dataease.provider.ProviderFactory; -import io.dataease.provider.query.QueryProvider; +import io.dataease.provider.QueryProvider; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -226,7 +226,7 @@ public class JdbcProvider extends DatasourceProvider { DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType()); switch (datasourceType) { case mysql: - case de_doris: + case engine_doris: case ds_doris: case mariadb: MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class); @@ -425,7 +425,7 @@ public class JdbcProvider extends DatasourceProvider { switch (datasourceType) { case mysql: case mariadb: - case de_doris: + case engine_doris: case ds_doris: MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class); username = mysqlConfiguration.getUsername(); @@ -529,7 +529,7 @@ public class JdbcProvider extends DatasourceProvider { switch (datasourceType) { case mysql: case mariadb: - case de_doris: + case engine_doris: case ds_doris: MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class); dataSource.setUrl(mysqlConfiguration.getJdbc()); @@ -607,7 +607,7 @@ public class JdbcProvider extends DatasourceProvider { case mariadb: JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class); return String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '%s' ;", jdbcConfiguration.getDataBase()); - case de_doris: + case engine_doris: case ds_doris: case hive: return "show tables"; @@ -656,7 +656,7 @@ public class JdbcProvider extends DatasourceProvider { switch (datasourceType) { case mysql: case mariadb: - case de_doris: + case engine_doris: case ds_doris: case ck: return null; diff --git a/backend/src/main/java/io/dataease/provider/query/doris/DorisConstants.java b/backend/src/main/java/io/dataease/provider/engine/doris/DorisConstants.java similarity index 74% rename from backend/src/main/java/io/dataease/provider/query/doris/DorisConstants.java rename to backend/src/main/java/io/dataease/provider/engine/doris/DorisConstants.java index fec288dee2..2305748abd 100644 --- a/backend/src/main/java/io/dataease/provider/query/doris/DorisConstants.java +++ b/backend/src/main/java/io/dataease/provider/engine/doris/DorisConstants.java @@ -1,17 +1,17 @@ -package io.dataease.provider.query.doris; +package io.dataease.provider.engine.doris; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.SQLConstants; -import static io.dataease.commons.constants.DatasourceTypes.de_doris; +import static io.dataease.commons.constants.DatasourceTypes.engine_doris; /** * @Author gin * @Date 2021/7/8 7:22 下午 */ public class DorisConstants extends SQLConstants { - public static final String KEYWORD_TABLE = de_doris.getKeywordPrefix() + "%s" + de_doris.getKeywordSuffix(); + public static final String KEYWORD_TABLE = engine_doris.getKeywordPrefix() + "%s" + engine_doris.getKeywordSuffix(); - public static final String KEYWORD_FIX = "%s." + de_doris.getKeywordPrefix() + "%s" + de_doris.getKeywordSuffix(); + public static final String KEYWORD_FIX = "%s." + engine_doris.getKeywordPrefix() + "%s" + engine_doris.getKeywordSuffix(); public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)"; diff --git a/backend/src/main/java/io/dataease/provider/engine/doris/DorisDDLProvider.java b/backend/src/main/java/io/dataease/provider/engine/doris/DorisDDLProvider.java new file mode 100644 index 0000000000..1d9965e2ca --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/engine/doris/DorisDDLProvider.java @@ -0,0 +1,85 @@ +package io.dataease.provider.engine.doris; + +import io.dataease.base.domain.DatasetTableField; +import io.dataease.commons.utils.TableUtils; +import io.dataease.provider.DDLProviderImpl; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * @Author gin + * @Date 2021/5/17 4:27 下午 + */ +@Service("dorisEngineDDL") +public class DorisDDLProvider extends DDLProviderImpl { + private static final String creatTableSql = "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" + + "Column_Fields" + + "UNIQUE KEY(dataease_uuid)\n" + + "DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" + + "PROPERTIES(\"replication_num\" = \"1\");"; + + @Override + public String createView(String name, String viewSQL) { + return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")"; + } + + @Override + public String dropTable(String name) { + return "DROP TABLE IF EXISTS " + name; + } + + @Override + public String dropView(String name) { + return "DROP VIEW IF EXISTS " + name; + } + + @Override + public String replaceTable(String name){ + return "ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false')" + .replace("DORIS_TABLE", name).replace("DORIS_TMP_TABLE", TableUtils.tmpName(name)); + } + + @Override + public String createTableSql(String tableName, List datasetTableFields) { + String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields); + return creatTableSql.replace("TABLE_NAME", tableName).replace("Column_Fields", dorisTableColumnSql); + } + + private String createDorisTableColumnSql(final List datasetTableFields) { + StringBuilder Column_Fields = new StringBuilder("dataease_uuid varchar(50), `"); + for (DatasetTableField datasetTableField : datasetTableFields) { + Column_Fields.append(datasetTableField.getDataeaseName()).append("` "); + Integer size = datasetTableField.getSize() * 3; + if (datasetTableField.getSize() == 0 || datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) { + size = 65533; + } + switch (datasetTableField.getDeExtractType()) { + case 0: + Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`"); + break; + case 1: + size = size < 50? 50 : size; + Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`"); + break; + case 2: + Column_Fields.append("bigint").append(",`"); + break; + case 3: + Column_Fields.append("DOUBLE").append(",`"); + break; + case 4: + Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(size))).append(",`"); + break; + default: + Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`"); + break; + } + } + + Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2)); + Column_Fields = new StringBuilder("(" + Column_Fields + ")\n"); + return Column_Fields.toString(); + } + +} diff --git a/backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java b/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java similarity index 99% rename from backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java rename to backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java index 4e2a59f65d..22b807936b 100644 --- a/backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java @@ -1,4 +1,4 @@ -package io.dataease.provider.query.doris; +package io.dataease.provider.engine.doris; import io.dataease.base.domain.ChartViewWithBLOBs; import io.dataease.base.domain.DatasetTableField; @@ -10,8 +10,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.QueryProvider; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -28,13 +28,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin * @Date 2021/5/17 4:11 下午 */ -@Service("dorisQuery") +@Service("dorisEngineQuery") public class DorisQueryProvider extends QueryProvider { @Resource private DatasetTableFieldMapper datasetTableFieldMapper; diff --git a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java new file mode 100644 index 0000000000..d2590b85f8 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java @@ -0,0 +1,49 @@ +package io.dataease.provider.engine.mysql; + +import io.dataease.provider.SQLConstants; + +import static io.dataease.commons.constants.DatasourceTypes.engine_mysql; + +/** + * @Author gin + * @Date 2021/7/8 7:22 下午 + */ +public class MysqlConstants extends SQLConstants { + public static final String KEYWORD_TABLE = engine_mysql.getKeywordPrefix() + "%s" + engine_mysql.getKeywordSuffix(); + + public static final String KEYWORD_FIX = "%s." + engine_mysql.getKeywordPrefix() + "%s" + engine_mysql.getKeywordSuffix(); + + public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)"; + + public static final String DATE_FORMAT = "DATE_FORMAT(%s,'%s')"; + + public static final String FROM_UNIXTIME = "FROM_UNIXTIME(%s,'%s')"; + + public static final String STR_TO_DATE = "STR_TO_DATE(%s,'%s')"; + + public static final String CAST = "CAST(%s AS %s)"; + + public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%S"; + + public static final String DEFAULT_INT_FORMAT = "BIGINT"; + + public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)"; + + public static final String WHERE_VALUE_NULL = "(NULL,'')"; + + public static final String WHERE_VALUE_VALUE = "'%s'"; + + public static final String WHERE_NUMBER_VALUE = "%s"; + + public static final String AGG_COUNT = "COUNT(*)"; + + public static final String AGG_FIELD = "%s(%s)"; + + public static final String WHERE_BETWEEN = "'%s' AND '%s'"; + + public static final String BRACKETS = "(%s)"; + + public static final String ROUND = "ROUND(%s,%s)"; + + public static final String VARCHAR = "VARCHAR"; +} diff --git a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java new file mode 100644 index 0000000000..61e7168d26 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java @@ -0,0 +1,35 @@ +package io.dataease.provider.engine.mysql; + +import io.dataease.commons.utils.TableUtils; +import io.dataease.provider.DDLProviderImpl; +import org.springframework.stereotype.Service; + +/** + * @Author gin + * @Date 2021/5/17 4:27 下午 + */ +@Service("mysqlEngineDDL") +public class MysqlDDLProvider extends DDLProviderImpl { + @Override + public String createView(String name, String viewSQL) { + return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")"; + } + + @Override + public String dropTable(String name) { + return "DROP TABLE IF EXISTS " + name; + } + + @Override + public String dropView(String name) { + return "DROP VIEW IF EXISTS " + name; + } + + @Override + public String replaceTable(String name){ + String replaceTableSql = "rename table FROM_TABLE to FROM_TABLE_tmp, TO_TABLE to FROM_TABLE, FROM_TABLE_tmp to TO_TABLE; " + .replace("FROM_TABLE", name).replace("TO_TABLE", TableUtils.tmpName(name)); + String dropTableSql = "DROP TABLE IF EXISTS " + TableUtils.tmpName(name); + return replaceTableSql + ";" + dropTableSql; + } +} diff --git a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java new file mode 100644 index 0000000000..9909543e8e --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java @@ -0,0 +1,1107 @@ +package io.dataease.provider.engine.mysql; + +import io.dataease.base.domain.ChartViewWithBLOBs; +import io.dataease.base.domain.DatasetTableField; +import io.dataease.base.domain.DatasetTableFieldExample; +import io.dataease.base.domain.Datasource; +import io.dataease.base.mapper.DatasetTableFieldMapper; +import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.dto.chart.ChartCustomFilterItemDTO; +import io.dataease.dto.chart.ChartFieldCustomFilterDTO; +import io.dataease.dto.chart.ChartViewFieldDTO; +import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STGroup; +import org.stringtemplate.v4.STGroupFile; + +import javax.annotation.Resource; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; + +/** + * @Author gin + * @Date 2021/5/17 4:11 下午 + */ +@Service("mysqlEngineQuery") +public class MysqlQueryProvider extends QueryProvider { + @Resource + private DatasetTableFieldMapper datasetTableFieldMapper; + + @Override + public Integer transFieldType(String field) { + switch (field) { + case "CHAR": + case "VARCHAR": + case "TEXT": + case "TINYTEXT": + case "MEDIUMTEXT": + case "LONGTEXT": + case "ENUM": + return 0;// 文本 + case "DATE": + case "TIME": + case "YEAR": + case "DATETIME": + case "TIMESTAMP": + return 1;// 时间 + case "INT": + case "SMALLINT": + case "MEDIUMINT": + case "INTEGER": + case "BIGINT": + return 2;// 整型 + case "FLOAT": + case "DOUBLE": + case "DECIMAL": + return 3;// 浮点 + case "BIT": + case "TINYINT": + return 4;// 布尔 + default: + return 0; + } + } + + @Override + public String createSQLPreview(String sql, String orderBy) { + return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp LIMIT 0,1000"; + } + + @Override + public String createQuerySQL(String table, List fields, boolean isGroup, Datasource ds, List fieldCustomFilter) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(fields)) { + for (int i = 0; i < fields.size(); i++) { + DatasetTableField f = fields.get(i); + String originField; + if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(f.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 1) { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getDataeaseName()); + } else { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getDataeaseName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + String fieldName = ""; + // 处理横轴字段 + if (f.getDeExtractType() == 1) { + if (f.getDeType() == 2 || f.getDeType() == 3) { + fieldName = String.format(MysqlConstants.UNIX_TIMESTAMP, originField) + "*1000"; + } else { + fieldName = originField; + } + } else if (f.getDeExtractType() == 0) { + if (f.getDeType() == 2) { + fieldName = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_INT_FORMAT); + } else if (f.getDeType() == 3) { + fieldName = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_FLOAT_FORMAT); + } else if (f.getDeType() == 1) { + fieldName = String.format(MysqlConstants.STR_TO_DATE, originField, MysqlConstants.DEFAULT_DATE_FORMAT); + } else { + fieldName = originField; + } + } else { + if (f.getDeType() == 1) { + String cast = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_INT_FORMAT) + "/1000"; + fieldName = String.format(MysqlConstants.FROM_UNIXTIME, cast, MysqlConstants.DEFAULT_DATE_FORMAT); + } else if (f.getDeType() == 2) { + fieldName = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_INT_FORMAT); + } else if (f.getDeType() == 3) { + fieldName = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_FLOAT_FORMAT); + } else { + fieldName = originField; + } + } + xFields.add(SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build()); + } + } + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("previewSql"); + st_sql.add("isGroup", isGroup); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + + return st_sql.render(); + } + + @Override + public String createQuerySQLAsTmp(String sql, List fields, boolean isGroup, List fieldCustomFilter) { + return createQuerySQL("(" + sql + ")", fields, isGroup, null, fieldCustomFilter); + } + + @Override + public String createQueryTableWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds, List fieldCustomFilter) { + return createQuerySQL(table, fields, isGroup, null, fieldCustomFilter) + " LIMIT " + (page - 1) * pageSize + "," + realSize; + } + + @Override + public String createQueryTableWithLimit(String table, List fields, Integer limit, boolean isGroup, Datasource ds, List fieldCustomFilter) { + return createQuerySQL(table, fields, isGroup, null, fieldCustomFilter) + " LIMIT 0," + limit; + } + + @Override + public String createQuerySqlWithLimit(String sql, List fields, Integer limit, boolean isGroup, List fieldCustomFilter) { + return createQuerySQLAsTmp(sql, fields, isGroup, fieldCustomFilter) + " LIMIT 0," + limit; + } + + @Override + public String createQuerySQLWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, List fieldCustomFilter) { + return createQuerySQLAsTmp(sql, fields, isGroup, fieldCustomFilter) + " LIMIT " + (page - 1) * pageSize + "," + realSize; + } + + @Override + public String getSQL(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName()); + } else { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName()); + } else { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.add(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + // 处理仪表板字段过滤 + String extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (extWheres != null) wheres.add(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList())); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MysqlConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return sqlLimit(st.render(), view); + } + + @Override + public String getSQLTableInfo(String table, List xAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName()); + } else { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + // 处理仪表板字段过滤 + String extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (extWheres != null) wheres.add(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("previewSql"); + st_sql.add("isGroup", false); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("previewSql"); + st.add("isGroup", false); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MysqlConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return sqlLimit(st.render(), view); + } + + @Override + public String getSQLAsTmpTableInfo(String sql, List xAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) { + return getSQLTableInfo("(" + sqlFix(sql) + ")", xAxis, fieldCustomFilter, extFilterRequestList, null, view); + } + + @Override + public String getSQLAsTmp(String sql, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) { + return getSQL("(" + sql + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, null, view); + } + + @Override + public String getSQLStack(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extStack, Datasource ds, ChartViewWithBLOBs view) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xOrders = new ArrayList<>(); + List xList = new ArrayList<>(); + xList.addAll(xAxis); + xList.addAll(extStack); + if (CollectionUtils.isNotEmpty(xList)) { + for (int i = 0; i < xList.size(); i++) { + ChartViewFieldDTO x = xList.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName()); + } else { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName()); + } else { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.add(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + // 处理仪表板字段过滤 + String extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (extWheres != null) wheres.add(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList())); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MysqlConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return sqlLimit(st.render(), view); + } + + @Override + public String getSQLAsTmpStack(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extStack, ChartViewWithBLOBs view) { + return getSQLStack("(" + table + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, extStack, null, view); + } + + @Override + public String getSQLScatter(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extBubble, Datasource ds, ChartViewWithBLOBs view) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName()); + } else { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + List yList = new ArrayList<>(); + yList.addAll(yAxis); + yList.addAll(extBubble); + if (CollectionUtils.isNotEmpty(yList)) { + for (int i = 0; i < yList.size(); i++) { + ChartViewFieldDTO y = yList.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName()); + } else { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.add(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + // 处理仪表板字段过滤 + String extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (extWheres != null) wheres.add(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList())); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MysqlConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return sqlLimit(st.render(), view); + } + + @Override + public String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extBubble, ChartViewWithBLOBs view) { + return getSQLScatter("(" + table + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, extBubble, null, view); + } + + @Override + public String searchTable(String table) { + return "SELECT table_name FROM information_schema.TABLES WHERE table_name ='" + table + "'"; + } + + @Override + public String getSQLSummary(String table, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) { + // 字段汇总 排序等 + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName()); + } else { + originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.add(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + // 处理仪表板字段过滤 + String extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (extWheres != null) wheres.add(extWheres); + List groups = new ArrayList<>(); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList())); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MysqlConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return sqlLimit(st.render(), view); + } + + @Override + public String getSQLSummaryAsTmp(String sql, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) { + return getSQLSummary("(" + sql + ")", yAxis, fieldCustomFilter, extFilterRequestList, view); + } + + @Override + public String wrapSql(String sql) { + sql = sql.trim(); + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0"; + return tmpSql; + } + + @Override + public String createRawQuerySQL(String table, List fields, Datasource ds) { + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("`").append(f.getDataeaseName()).append("` AS ").append(f.getDataeaseName()); + return stringBuilder.toString(); + }).toArray(String[]::new); + return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table); + } + + @Override + public String createRawQuerySQLAsTmp(String sql, List fields) { + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); + } + + @Override + public String convertTableToSql(String tableName, Datasource ds) { + return createSQLPreview("SELECT * FROM " + String.format(MysqlConstants.KEYWORD_TABLE, tableName), null); + } + + private String sqlFix(String sql) { + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + return sql; + } + + public String transMysqlFilterTerm(String term) { + switch (term) { + case "eq": + return " = "; + case "not_eq": + return " <> "; + case "lt": + return " < "; + case "le": + return " <= "; + case "gt": + return " > "; + case "ge": + return " >= "; + case "in": + return " IN "; + case "not in": + return " NOT IN "; + case "like": + return " LIKE "; + case "not like": + return " NOT LIKE "; + case "null": + return " IS NULL "; + case "not_null": + return " IS NOT NULL "; + case "empty": + return " = "; + case "not_empty": + return " <> "; + case "between": + return " BETWEEN "; + default: + return ""; + } + } + + public String transCustomFilterList(SQLObj tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + List res = new ArrayList<>(); + + for (ChartFieldCustomFilterDTO request : requestList) { + List list = new ArrayList<>(); + DatasetTableField field = request.getField(); + + if (ObjectUtils.isEmpty(field)) { + continue; + } + String whereName = ""; + String originName; + if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originName = calcFieldRegex(field.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { + originName = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getDataeaseName()); + } else { + originName = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getDataeaseName()); + } + + if (field.getDeType() == 1) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5 || field.getDeExtractType() == 1) { + whereName = String.format(MysqlConstants.STR_TO_DATE, originName, MysqlConstants.DEFAULT_DATE_FORMAT); + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + String cast = String.format(MysqlConstants.CAST, originName, MysqlConstants.DEFAULT_INT_FORMAT) + "/1000"; + whereName = String.format(MysqlConstants.FROM_UNIXTIME, cast, MysqlConstants.DEFAULT_DATE_FORMAT); + } + } else if (field.getDeType() == 2 || field.getDeType() == 3) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + whereName = String.format(MysqlConstants.CAST, originName, MysqlConstants.DEFAULT_FLOAT_FORMAT); + } + if (field.getDeExtractType() == 1) { + whereName = String.format(MysqlConstants.UNIX_TIMESTAMP, originName) + "*1000"; + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + whereName = originName; + } + } else if (field.getDeType() == 0) { + whereName = String.format(MysqlConstants.CAST, originName, MysqlConstants.VARCHAR); + } else { + whereName = originName; + } + + if (StringUtils.equalsIgnoreCase(request.getFilterType(), "enum")) { + if (CollectionUtils.isNotEmpty(request.getEnumCheckField())) { + res.add("(" + whereName + " IN ('" + String.join("','", request.getEnumCheckField()) + "'))"); + } + } else { + List filter = request.getFilter(); + for (ChartCustomFilterItemDTO filterItemDTO : filter) { + String value = filterItemDTO.getValue(); + String whereTerm = transMysqlFilterTerm(filterItemDTO.getTerm()); + String whereValue = ""; + + if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "empty")) { + whereValue = "''"; + } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) { + whereValue = "''"; + } else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) { + whereValue = "'%" + value + "%'"; + } else { + // Doris field type test + /*if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + whereValue = String.format(DorisConstants.WHERE_NUMBER_VALUE, value); + } else { + whereValue = String.format(DorisConstants.WHERE_VALUE_VALUE, value); + }*/ + whereValue = String.format(MysqlConstants.WHERE_VALUE_VALUE, value); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); + } + + List strList = new ArrayList<>(); + list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue())); + if (CollectionUtils.isNotEmpty(list)) { + res.add("(" + String.join(" " + getLogic(request.getLogic()) + " ", strList) + ")"); + } + } + } + return CollectionUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null; + } + + public String transExtFilterList(SQLObj tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + List list = new ArrayList<>(); + for (ChartExtFilterRequest request : requestList) { + List value = request.getValue(); + DatasetTableField field = request.getDatasetTableField(); + if (CollectionUtils.isEmpty(value) || ObjectUtils.isEmpty(field)) { + continue; + } + String whereName = ""; + String whereTerm = transMysqlFilterTerm(request.getOperator()); + String whereValue = ""; + + String originName; + if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originName = calcFieldRegex(field.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { + originName = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getDataeaseName()); + } else { + originName = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getDataeaseName()); + } + + if (field.getDeType() == 1) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5 || field.getDeExtractType() == 1) { + whereName = String.format(MysqlConstants.STR_TO_DATE, originName, MysqlConstants.DEFAULT_DATE_FORMAT); + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + String cast = String.format(MysqlConstants.CAST, originName, MysqlConstants.DEFAULT_INT_FORMAT) + "/1000"; + whereName = String.format(MysqlConstants.FROM_UNIXTIME, cast, MysqlConstants.DEFAULT_DATE_FORMAT); + } + } else if (field.getDeType() == 0) { + whereName = String.format(MysqlConstants.CAST, originName, MysqlConstants.VARCHAR); + } else if (field.getDeType() == 2 || field.getDeType() == 3) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + whereName = String.format(MysqlConstants.CAST, originName, MysqlConstants.DEFAULT_FLOAT_FORMAT); + } + if (field.getDeExtractType() == 1) { + whereName = String.format(MysqlConstants.UNIX_TIMESTAMP, originName) + "*1000"; + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + whereName = originName; + } + } else { + whereName = originName; + } + + + if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { + whereValue = "'%" + value.get(0) + "%'"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) { + if (request.getDatasetTableField().getDeType() == 1) { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0)))); + String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1)))); + whereValue = String.format(MysqlConstants.WHERE_BETWEEN, startTime, endTime); + } else { + whereValue = String.format(MysqlConstants.WHERE_BETWEEN, value.get(0), value.get(1)); + } + } else { + // doris field type test + /*if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + whereValue = String.format(DorisConstants.WHERE_NUMBER_VALUE, value.get(0)); + } else { + whereValue = String.format(DorisConstants.WHERE_VALUE_VALUE, value.get(0)); + }*/ + whereValue = String.format(MysqlConstants.WHERE_VALUE_VALUE, value.get(0)); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); + } + List strList = new ArrayList<>(); + list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue())); + return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" AND ", strList) + ")" : null; + } + + private String transDateFormat(String dateStyle, String datePattern) { + String split = "-"; + if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) { + split = "-"; + } else if (StringUtils.equalsIgnoreCase(datePattern, "date_split")) { + split = "/"; + } else { + split = "-"; + } + + if (StringUtils.isEmpty(dateStyle)) { + return "%Y-%m-%d %H:%i:%S"; + } + + switch (dateStyle) { + case "y": + return "%Y"; + case "y_M": + return "%Y" + split + "%m"; + case "y_M_d": + return "%Y" + split + "%m" + split + "%d"; + case "H_m_s": + return "%H:%i:%S"; + case "y_M_d_H_m": + return "%Y" + split + "%m" + split + "%d" + " %H:%i"; + case "y_M_d_H_m_s": + return "%Y" + split + "%m" + split + "%d" + " %H:%i:%S"; + default: + return "%Y-%m-%d %H:%i:%S"; + } + } + + private SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) { + String fieldName = ""; + if (x.getDeExtractType() == 1) { + if (x.getDeType() == 2 || x.getDeType() == 3) { + fieldName = String.format(MysqlConstants.UNIX_TIMESTAMP, originField) + "*1000"; + } else if (x.getDeType() == 1) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + fieldName = String.format(MysqlConstants.DATE_FORMAT, originField, format); + } else { + fieldName = originField; + } + } else { + if (x.getDeType() == 1) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + if (x.getDeExtractType() == 0) { + fieldName = String.format(MysqlConstants.DATE_FORMAT, originField, format); + } else { + String cast = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_INT_FORMAT) + "/1000"; + String from_unixtime = String.format(MysqlConstants.FROM_UNIXTIME, cast, MysqlConstants.DEFAULT_DATE_FORMAT); + fieldName = String.format(MysqlConstants.DATE_FORMAT, from_unixtime, format); + } + } else if (x.getDeType() == 0) { + fieldName = String.format(MysqlConstants.CAST, originField, MysqlConstants.VARCHAR); + } else { + fieldName = originField; + } + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private List getXWheres(ChartViewFieldDTO x, String originField, String fieldAlias) { + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) { + x.getFilter().forEach(f -> { + String whereName = ""; + String whereTerm = transMysqlFilterTerm(f.getTerm()); + String whereValue = ""; + if (x.getDeType() == 1 && x.getDeExtractType() != 1) { + String cast = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_INT_FORMAT) + "/1000"; + whereName = String.format(MysqlConstants.FROM_UNIXTIME, cast, MysqlConstants.DEFAULT_DATE_FORMAT); + } else { + whereName = originField; + } + if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "empty")) { + whereValue = "''"; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_empty")) { + whereValue = "''"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + whereValue = "'%" + f.getValue() + "%'"; + } else { + whereValue = String.format(MysqlConstants.WHERE_VALUE_VALUE, f.getValue()); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereAlias(fieldAlias) + .whereTermAndValue(whereTerm + whereValue) + .build()); + }); + } + return list; + } + + private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) { + String fieldName = ""; + if (StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*")) { + fieldName = MysqlConstants.AGG_COUNT; + } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) { + fieldName = String.format(MysqlConstants.AGG_FIELD, y.getSummary(), originField); + } else { + if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { + String cast = String.format(MysqlConstants.CAST, originField, y.getDeType() == 2 ? MysqlConstants.DEFAULT_INT_FORMAT : MysqlConstants.DEFAULT_FLOAT_FORMAT); + String agg = String.format(MysqlConstants.AGG_FIELD, y.getSummary(), cast); + String cast1 = String.format(MysqlConstants.CAST, agg, MysqlConstants.DEFAULT_FLOAT_FORMAT); + fieldName = String.format(MysqlConstants.ROUND, cast1, "2"); + } else { + String cast = String.format(MysqlConstants.CAST, originField, y.getDeType() == 2 ? MysqlConstants.DEFAULT_INT_FORMAT : MysqlConstants.DEFAULT_FLOAT_FORMAT); + fieldName = String.format(MysqlConstants.AGG_FIELD, y.getSummary(), cast); + } + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private String getYWheres(ChartViewFieldDTO y, String originField, String fieldAlias) { + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) { + y.getFilter().forEach(f -> { + String whereTerm = transMysqlFilterTerm(f.getTerm()); + String whereValue = ""; + // 原始类型不是时间,在de中被转成时间的字段做处理 + if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "empty")) { + whereValue = "''"; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_empty")) { + whereValue = "''"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + whereValue = "'%" + f.getValue() + "%'"; + } else { + whereValue = String.format(MysqlConstants.WHERE_VALUE_VALUE, f.getValue()); + } + list.add(SQLObj.builder() + .whereField(fieldAlias) + .whereAlias(fieldAlias) + .whereTermAndValue(whereTerm + whereValue) + .build()); + }); + } + List strList = new ArrayList<>(); + list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue())); + return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" " + getLogic(y.getLogic()) + " ", strList) + ")" : null; + } + + private String calcFieldRegex(String originField, SQLObj tableObj) { + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getDataeaseName())); + } + return originField; + } + + private String sqlLimit(String sql, ChartViewWithBLOBs view) { + if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) { + return sql + " LIMIT 0," + view.getResultCount(); + } else { + return sql; + } + } +} diff --git a/backend/src/main/java/io/dataease/provider/query/DDLProvider.java b/backend/src/main/java/io/dataease/provider/query/DDLProvider.java deleted file mode 100644 index ba1e218c1e..0000000000 --- a/backend/src/main/java/io/dataease/provider/query/DDLProvider.java +++ /dev/null @@ -1,13 +0,0 @@ -package io.dataease.provider.query; - -/** - * @Author gin - * @Date 2021/5/17 4:19 下午 - */ -public abstract class DDLProvider { - public abstract String createView(String name, String viewSQL); - - public abstract String dropTable(String name); - - public abstract String dropView(String name); -} diff --git a/backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java b/backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java index 80bfe8f3c4..8f9038662e 100644 --- a/backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java @@ -1,6 +1,6 @@ package io.dataease.provider.query.api; -import io.dataease.provider.query.QueryProviderImpl; +import io.dataease.provider.QueryProviderImpl; import org.springframework.stereotype.Service; @Service("apiQuery") diff --git a/backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java b/backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java index 173849d756..7f9bd5d9ec 100644 --- a/backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java +++ b/backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java @@ -1,6 +1,6 @@ package io.dataease.provider.query.ck; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.ck; diff --git a/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java index c1165b2535..8f5d32a266 100644 --- a/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java @@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.QueryProvider; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -29,7 +29,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java b/backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java index 307d166243..e914d91b9c 100644 --- a/backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java +++ b/backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java @@ -1,6 +1,6 @@ package io.dataease.provider.query.db2; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.db2; diff --git a/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java b/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java index a1593afca8..d9fd103ce0 100644 --- a/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java @@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.datasource.Db2Configuration; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.QueryProvider; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -31,7 +31,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; @Service("db2Query") public class Db2QueryProvider extends QueryProvider { diff --git a/backend/src/main/java/io/dataease/provider/query/doris/DorisDDLProvider.java b/backend/src/main/java/io/dataease/provider/query/doris/DorisDDLProvider.java deleted file mode 100644 index cfa3f420a7..0000000000 --- a/backend/src/main/java/io/dataease/provider/query/doris/DorisDDLProvider.java +++ /dev/null @@ -1,26 +0,0 @@ -package io.dataease.provider.query.doris; - -import io.dataease.provider.query.DDLProvider; -import org.springframework.stereotype.Service; - -/** - * @Author gin - * @Date 2021/5/17 4:27 下午 - */ -@Service("dorisDDL") -public class DorisDDLProvider extends DDLProvider { - @Override - public String createView(String name, String viewSQL) { - return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")"; - } - - @Override - public String dropTable(String name) { - return "DROP TABLE IF EXISTS " + name; - } - - @Override - public String dropView(String name) { - return "DROP VIEW IF EXISTS " + name; - } -} diff --git a/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java index 77f24131ab..95fb323686 100644 --- a/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java @@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.QueryProvider; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -29,7 +29,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; @Service("esQuery") public class EsQueryProvider extends QueryProvider { diff --git a/backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java b/backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java index 62301f3bf6..a9186a68e7 100644 --- a/backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java +++ b/backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java @@ -1,6 +1,6 @@ package io.dataease.provider.query.es; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.es; diff --git a/backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java b/backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java index 4e99ae88e8..ec1a82007f 100644 --- a/backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java +++ b/backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java @@ -1,6 +1,6 @@ package io.dataease.provider.query.hive; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.mysql; diff --git a/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java index 457651b642..9e054efe3f 100644 --- a/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java @@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.QueryProvider; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -29,7 +29,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java index 0d4f8127de..14a7107f96 100644 --- a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java +++ b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java @@ -1,6 +1,6 @@ package io.dataease.provider.query.mongodb; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.mongo; diff --git a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java index e2922121c0..97f7bcf7f1 100644 --- a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java @@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.QueryProvider; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -29,7 +29,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java b/backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java index 160ff281c9..33b1e3d8ea 100644 --- a/backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java +++ b/backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java @@ -1,6 +1,6 @@ package io.dataease.provider.query.mysql; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.mysql; diff --git a/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java index 706c0fe034..95f653fb12 100644 --- a/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java @@ -10,8 +10,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.QueryProvider; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -28,7 +28,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java b/backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java index c8e500e174..013e51defc 100644 --- a/backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java +++ b/backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java @@ -1,6 +1,6 @@ package io.dataease.provider.query.oracle; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.oracle; diff --git a/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java index 3fc25eb97d..48d04457c5 100644 --- a/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java @@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.datasource.JdbcConfiguration; import io.dataease.dto.datasource.OracleConfiguration; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.QueryProvider; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -31,7 +31,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java b/backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java index 25c50a431f..d0e9f7cdeb 100644 --- a/backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java +++ b/backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java @@ -1,6 +1,6 @@ package io.dataease.provider.query.pg; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.pg; diff --git a/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java index 55eebb6755..b231d74536 100644 --- a/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java @@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.datasource.JdbcConfiguration; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.QueryProvider; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; import io.dataease.provider.query.sqlserver.SqlServerSQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; @@ -32,7 +32,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; @Service("pgQuery") diff --git a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java index f52c1d36d2..f1ba7179aa 100644 --- a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java +++ b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java @@ -1,6 +1,6 @@ package io.dataease.provider.query.redshift; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.pg; diff --git a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java index b3f54f14ff..3067caaf8e 100644 --- a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java @@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.datasource.JdbcConfiguration; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.QueryProvider; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; import io.dataease.provider.query.pg.PgConstants; import io.dataease.provider.query.sqlserver.SqlServerSQLConstants; import org.apache.commons.collections4.CollectionUtils; @@ -33,7 +33,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; /** diff --git a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java b/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java index 1817f77afa..ac5862abfa 100644 --- a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java +++ b/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java @@ -1,6 +1,6 @@ package io.dataease.provider.query.sqlserver; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.sqlServer; diff --git a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java index 99f855c936..2718468fdf 100644 --- a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java @@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.datasource.JdbcConfiguration; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.QueryProvider; -import io.dataease.provider.query.SQLConstants; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -31,7 +31,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; @Service("sqlserverQuery") public class SqlserverQueryProvider extends QueryProvider { diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index 76924fd0c2..69778f5f6e 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -2,7 +2,6 @@ package io.dataease.service.chart; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; -import io.dataease.auth.api.dto.CurrentUserDto; import io.dataease.auth.entity.SysUserEntity; import io.dataease.auth.service.AuthUserService; import io.dataease.base.domain.*; @@ -15,7 +14,6 @@ import io.dataease.commons.constants.CommonConstants; import io.dataease.commons.constants.JdbcConstants; import io.dataease.commons.utils.AuthUtils; import io.dataease.commons.utils.BeanUtils; -import io.dataease.commons.utils.CommonBeanFactory; import io.dataease.commons.utils.LogUtil; import io.dataease.controller.request.chart.*; import io.dataease.controller.request.datasource.DatasourceRequest; @@ -29,12 +27,13 @@ import io.dataease.i18n.Translator; import io.dataease.listener.util.CacheUtils; import io.dataease.provider.ProviderFactory; import io.dataease.provider.datasource.DatasourceProvider; -import io.dataease.provider.query.QueryProvider; +import io.dataease.provider.QueryProvider; import io.dataease.service.dataset.DataSetTableFieldsService; import io.dataease.service.dataset.DataSetTableService; import io.dataease.service.dataset.DataSetTableUnionService; import io.dataease.service.dataset.PermissionService; import io.dataease.service.datasource.DatasourceService; +import io.dataease.service.engine.EngineService; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -74,6 +73,8 @@ public class ChartViewService { @Resource private AuthUserService authUserService; @Resource + private EngineService engineService; + @Resource private ChartViewCacheMapper chartViewCacheMapper; //默认使用非公平 @@ -530,7 +531,7 @@ public class ChartViewService { data = datasourceProvider.getData(datasourceRequest); } else if (table.getMode() == 1) {// 抽取 // 连接doris,构建doris数据源查询 - Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource ds = engineService.getDeEngine(); DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); datasourceRequest.setDatasource(ds); String tableName = "ds_" + table.getId().replaceAll("-", "_"); diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java index 4744c14637..185e68a6a5 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java @@ -4,7 +4,7 @@ import io.dataease.base.domain.DatasetTableField; import io.dataease.base.domain.DatasetTableFieldExample; import io.dataease.base.mapper.DatasetTableFieldMapper; import io.dataease.commons.exception.DEException; -import io.dataease.commons.utils.DorisTableUtils; +import io.dataease.commons.utils.TableUtils; import io.dataease.i18n.Translator; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; @@ -36,7 +36,7 @@ public class DataSetTableFieldsService { datasetTableField.setId(UUID.randomUUID().toString()); // 若dataeasename为空,则用MD5(id)作为dataeasename if (StringUtils.isEmpty(datasetTableField.getDataeaseName())) { - datasetTableField.setDataeaseName(DorisTableUtils.columnName(datasetTableField.getId())); + datasetTableField.setDataeaseName(TableUtils.columnName(datasetTableField.getId())); } if (ObjectUtils.isEmpty(datasetTableField.getLastSyncTime())) { datasetTableField.setLastSyncTime(System.currentTimeMillis()); diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index e95faa427a..64ade01908 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -28,8 +28,9 @@ import io.dataease.plugins.loader.ClassloaderResponsity; import io.dataease.provider.ProviderFactory; import io.dataease.provider.datasource.DatasourceProvider; import io.dataease.provider.datasource.JdbcProvider; -import io.dataease.provider.query.DDLProvider; -import io.dataease.provider.query.QueryProvider; +import io.dataease.provider.DDLProvider; +import io.dataease.provider.QueryProvider; +import io.dataease.service.engine.EngineService; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -97,6 +98,8 @@ public class DataSetTableService { private DatasetTableFieldMapper datasetTableFieldMapper; @Resource private PermissionService permissionService; + @Resource + private EngineService engineService; private static final String lastUpdateTime = "${__last_update_time__}"; private static final String currentUpdateTime = "${__current_update_time__}"; @@ -296,8 +299,8 @@ public class DataSetTableService { } private void deleteDorisTable(String datasetId, DatasetTable table) throws Exception { - String dorisTableName = DorisTableUtils.dorisName(datasetId); - Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + String dorisTableName = TableUtils.tableName(datasetId); + Datasource dorisDatasource = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(dorisDatasource); @@ -306,12 +309,12 @@ public class DataSetTableService { || StringUtils.equalsIgnoreCase("union", table.getType())) { datasourceRequest.setQuery(ddlProvider.dropView(dorisTableName)); jdbcProvider.exec(datasourceRequest); - datasourceRequest.setQuery(ddlProvider.dropView(DorisTableUtils.dorisTmpName(dorisTableName))); + datasourceRequest.setQuery(ddlProvider.dropView(TableUtils.tmpName(dorisTableName))); jdbcProvider.exec(datasourceRequest); } else { datasourceRequest.setQuery(ddlProvider.dropTable(dorisTableName)); jdbcProvider.exec(datasourceRequest); - datasourceRequest.setQuery(ddlProvider.dropTable(DorisTableUtils.dorisTmpName(dorisTableName))); + datasourceRequest.setQuery(ddlProvider.dropTable(TableUtils.tmpName(dorisTableName))); jdbcProvider.exec(datasourceRequest); } } @@ -551,11 +554,11 @@ public class DataSetTableService { if (!checkDorisTableIsExists(dataSetTableRequest.getId())) { throw new RuntimeException(Translator.get("i18n_data_not_sync")); } - Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource ds = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); + String table = TableUtils.tableName(dataSetTableRequest.getId()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); datasourceRequest.setQuery( qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter)); @@ -620,11 +623,11 @@ public class DataSetTableService { if (!checkDorisTableIsExists(dataSetTableRequest.getId())) { throw new RuntimeException(Translator.get("i18n_data_not_sync")); } - Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource ds = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); + String table = TableUtils.tableName(dataSetTableRequest.getId()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); datasourceRequest.setQuery( qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter)); @@ -649,11 +652,11 @@ public class DataSetTableService { throw new RuntimeException(Translator.get("i18n_data_not_sync")); } - Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource ds = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); + String table = TableUtils.tableName(dataSetTableRequest.getId()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); datasourceRequest.setQuery( qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter)); @@ -719,11 +722,11 @@ public class DataSetTableService { DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage()); } } else { - Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource ds = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); + String table = TableUtils.tableName(dataSetTableRequest.getId()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); datasourceRequest.setQuery( qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter)); @@ -789,11 +792,11 @@ public class DataSetTableService { DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage()); } } else { - Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource ds = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); + String table = TableUtils.tableName(dataSetTableRequest.getId()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); datasourceRequest.setQuery( qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter)); @@ -889,7 +892,7 @@ public class DataSetTableService { datasourceRequest.setDatasource(ds); sqlMap = getUnionSQLDatasource(dataTableInfoDTO, ds); } else { - ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + ds = engineService.getDeEngine(); datasourceRequest.setDatasource(ds); sqlMap = getUnionSQLDoris(dataTableInfoDTO); } @@ -923,10 +926,10 @@ public class DataSetTableService { for (DatasetTableField datasetTableField : fieldList) { for (TableField tableField : fields) { if (StringUtils.equalsIgnoreCase(tableField.getFieldName(), - DorisTableUtils.dorisFieldName( + TableUtils.fieldName( datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName())) || StringUtils.equalsIgnoreCase(tableField.getFieldName(), - DorisTableUtils.dorisFieldNameShort(datasetTableField.getTableId() + "_" + TableUtils.fieldNameShort(datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) { tableField.setRemarks(datasetTableField.getName()); break; @@ -955,7 +958,7 @@ public class DataSetTableService { datasourceRequest.setDatasource(ds); sql = getCustomSQLDatasource(dataTableInfoDTO, list, ds); } else { - ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + ds = engineService.getDeEngine(); datasourceRequest.setDatasource(ds); sql = getCustomSQLDoris(dataTableInfoDTO, list); } @@ -987,9 +990,9 @@ public class DataSetTableService { for (DatasetTableField datasetTableField : checkedFieldList) { for (TableField tableField : fields) { if (StringUtils.equalsIgnoreCase(tableField.getFieldName(), - DorisTableUtils.dorisFieldName( + TableUtils.fieldName( datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName())) - || StringUtils.equalsIgnoreCase(tableField.getFieldName(), DorisTableUtils.dorisFieldName( + || StringUtils.equalsIgnoreCase(tableField.getFieldName(), TableUtils.fieldName( datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) { tableField.setRemarks(datasetTableField.getName()); break; @@ -1009,7 +1012,7 @@ public class DataSetTableService { private String getCustomSQLDoris(DataTableInfoDTO dataTableInfoDTO, List list) { Map customInfo = new TreeMap<>(); dataTableInfoDTO.getList().forEach(ele -> { - String table = DorisTableUtils.dorisName(ele.getTableId()); + String table = TableUtils.tableName(ele.getTableId()); DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(ele.getTableId()); if (ObjectUtils.isEmpty(datasetTable)) { throw new RuntimeException(Translator.get("i18n_custom_ds_delete")); @@ -1020,7 +1023,7 @@ public class DataSetTableService { } String[] array = fields.stream() .map(f -> table + "." + f.getDataeaseName() + " AS " - + DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName())) + + TableUtils.fieldName(ele.getTableId() + "_" + f.getDataeaseName())) .toArray(String[]::new); customInfo.put(table, array); }); @@ -1046,12 +1049,12 @@ public class DataSetTableService { } join.append(convertUnionTypeToSQL(dto.getSourceUnionRelation())) - .append(DorisTableUtils.dorisName(dto.getTargetTableId())) + .append(TableUtils.tableName(dto.getTargetTableId())) .append(" ON ") - .append(DorisTableUtils.dorisName(dto.getSourceTableId())).append(".") + .append(TableUtils.tableName(dto.getSourceTableId())).append(".") .append(sourceField.getDataeaseName()) .append(" = ") - .append(DorisTableUtils.dorisName(dto.getTargetTableId())).append(".") + .append(TableUtils.tableName(dto.getTargetTableId())).append(".") .append(targetField.getDataeaseName()); } } @@ -1059,16 +1062,16 @@ public class DataSetTableService { if (StringUtils.isEmpty(f)) { throw new RuntimeException(Translator.get("i18n_custom_ds_delete")); } - return MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(first.getTableId())) + return MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.tableName(first.getTableId())) + join.toString(); } else { if (StringUtils - .isEmpty(StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","))) { + .isEmpty(StringUtils.join(customInfo.get(TableUtils.tableName(first.getTableId())), ","))) { throw new RuntimeException(Translator.get("i18n_custom_ds_delete")); } return MessageFormat.format("SELECT {0} FROM {1}", - StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","), - DorisTableUtils.dorisName(first.getTableId())); + StringUtils.join(customInfo.get(TableUtils.tableName(first.getTableId())), ","), + TableUtils.tableName(first.getTableId())); } } @@ -1089,7 +1092,7 @@ public class DataSetTableService { } String[] array = fields.stream() .map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS " - + DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getOriginName())) + + TableUtils.fieldName(ele.getTableId() + "_" + f.getOriginName())) .toArray(String[]::new); customInfo.put(table, array); } @@ -1175,7 +1178,7 @@ public class DataSetTableService { for (UnionDTO unionDTO : union) { // doris 使用数据集id做表名,拼sql将用到该名称 String tableId = unionDTO.getCurrentDs().getId(); - String table = DorisTableUtils.dorisName(tableId); + String table = TableUtils.tableName(tableId); DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId); if (ObjectUtils.isEmpty(datasetTable)) { DEException.throwException( @@ -1193,7 +1196,7 @@ public class DataSetTableService { String[] array = fields.stream() .map(f -> table + "." + f.getDataeaseName() + " AS " - + DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName())) + + TableUtils.fieldName(tableId + "_" + f.getDataeaseName())) .toArray(String[]::new); checkedInfo.put(table, array); checkedFields.addAll(fields); @@ -1225,7 +1228,7 @@ public class DataSetTableService { DatasetTable parentTable = datasetTableMapper.selectByPrimaryKey(pField.getTableId()); DatasetTable currentTable = datasetTableMapper.selectByPrimaryKey(cField.getTableId()); - join.append(" ").append(joinType).append(" ").append(DorisTableUtils.dorisName(currentTable.getId())) + join.append(" ").append(joinType).append(" ").append(TableUtils.tableName(currentTable.getId())) .append(" ON "); for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) { UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i); @@ -1235,10 +1238,10 @@ public class DataSetTableService { DatasetTableField currentField = dataSetTableFieldsService .get(unionItemDTO.getCurrentField().getId()); - join.append(DorisTableUtils.dorisName(parentTable.getId())).append(".") + join.append(TableUtils.tableName(parentTable.getId())).append(".") .append(parentField.getDataeaseName()) .append(" = ") - .append(DorisTableUtils.dorisName(currentTable.getId())).append(".") + .append(TableUtils.tableName(currentTable.getId())).append(".") .append(currentField.getDataeaseName()); if (i < unionParamDTO.getUnionFields().size() - 1) { join.append(" AND "); @@ -1249,15 +1252,15 @@ public class DataSetTableService { DEException.throwException(Translator.get("i18n_union_ds_no_checked")); } sql = MessageFormat.format("SELECT {0} FROM {1}", f, - DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())) + join.toString(); + TableUtils.tableName(union.get(0).getCurrentDs().getId())) + join.toString(); } else { - String f = StringUtils.join(checkedInfo.get(DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())), + String f = StringUtils.join(checkedInfo.get(TableUtils.tableName(union.get(0).getCurrentDs().getId())), ","); if (StringUtils.isEmpty(f)) { throw new RuntimeException(Translator.get("i18n_union_ds_no_checked")); } sql = MessageFormat.format("SELECT {0} FROM {1}", f, - DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())); + TableUtils.tableName(union.get(0).getCurrentDs().getId())); } Map map = new HashMap<>(); map.put("sql", sql); @@ -1272,7 +1275,7 @@ public class DataSetTableService { for (int i = 0; i < childrenDs.size(); i++) { UnionDTO unionDTO = childrenDs.get(i); String tableId = unionDTO.getCurrentDs().getId(); - String table = DorisTableUtils.dorisName(tableId); + String table = TableUtils.tableName(tableId); DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId); if (ObjectUtils.isEmpty(datasetTable)) { DEException.throwException( @@ -1282,7 +1285,7 @@ public class DataSetTableService { String[] array = fields.stream() .map(f -> table + "." + f.getDataeaseName() + " AS " - + DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName())) + + TableUtils.fieldName(tableId + "_" + f.getDataeaseName())) .toArray(String[]::new); checkedInfo.put(table, array); checkedFields.addAll(fields); @@ -1329,7 +1332,7 @@ public class DataSetTableService { String[] array = fields.stream() .map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS " - + DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName())) + + TableUtils.fieldNameShort(tableId + "_" + f.getOriginName())) .toArray(String[]::new); checkedInfo.put(table, array); checkedFields.addAll(fields); @@ -1430,7 +1433,7 @@ public class DataSetTableService { String[] array = fields.stream() .map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS " - + DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName())) + + TableUtils.fieldNameShort(tableId + "_" + f.getOriginName())) .toArray(String[]::new); checkedInfo.put(table, array); checkedFields.addAll(fields); @@ -1451,7 +1454,7 @@ public class DataSetTableService { datasetTableField.setTableId(datasetTableId); datasetTableField.setOriginName(filed.getFieldName()); datasetTableField.setName(filed.getRemarks()); - datasetTableField.setDataeaseName(DorisTableUtils.columnName(filed.getFieldName())); + datasetTableField.setDataeaseName(TableUtils.columnName(filed.getFieldName())); datasetTableField.setType(filed.getFieldType()); datasetTableField.setDeType(transFieldType(filed.getFieldType())); datasetTableField.setDeExtractType(transFieldType(filed.getFieldType())); @@ -1501,7 +1504,7 @@ public class DataSetTableService { List listByIds = dataSetTableFieldsService .getListByIdsEach(ele.getCheckedFields()); listByIds.forEach(f -> f.setDataeaseName( - DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName()))); + TableUtils.fieldName(ele.getTableId() + "_" + f.getDataeaseName()))); fieldList.addAll(listByIds); }); for (int i = 0; i < fieldList.size(); i++) { @@ -1514,7 +1517,7 @@ public class DataSetTableService { dataSetTableFieldsService.batchEdit(fieldList); // custom 创建doris视图 if (datasetTable.getMode() == 1) { - createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), getCustomSQLDoris(dataTableInfoDTO, + createDorisView(TableUtils.tableName(datasetTable.getId()), getCustomSQLDoris(dataTableInfoDTO, dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId()))); } return; @@ -1541,7 +1544,7 @@ public class DataSetTableService { for (DatasetTableField field : fieldList) { for (TableField tableField : fields) { if (StringUtils.equalsIgnoreCase( - DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getOriginName()), + TableUtils.fieldName(field.getTableId() + "_" + field.getOriginName()), tableField.getFieldName())) { tableField.setRemarks(field.getName()); break; @@ -1551,7 +1554,7 @@ public class DataSetTableService { } } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "union")) { if (datasetTable.getMode() == 1) { - ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + ds = engineService.getDeEngine(); DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); @@ -1564,14 +1567,14 @@ public class DataSetTableService { List join = (List) sqlMap.get("join"); // custom 创建doris视图 - createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), sql); + createDorisView(TableUtils.tableName(datasetTable.getId()), sql); datasourceRequest.setQuery(sql); fields = datasourceProvider.fetchResultField(datasourceRequest); for (DatasetTableField field : fieldList) { for (TableField tableField : fields) { if (StringUtils.equalsIgnoreCase( - DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getDataeaseName()), + TableUtils.fieldName(field.getTableId() + "_" + field.getDataeaseName()), tableField.getFieldName())) { tableField.setRemarks(field.getName()); break; @@ -1595,7 +1598,7 @@ public class DataSetTableService { for (DatasetTableField field : fieldList) { for (TableField tableField : fields) { if (StringUtils.equalsIgnoreCase( - DorisTableUtils.dorisFieldNameShort(field.getTableId() + "_" + field.getOriginName()), + TableUtils.fieldNameShort(field.getTableId() + "_" + field.getOriginName()), tableField.getFieldName())) { tableField.setRemarks(field.getName()); break; @@ -1637,7 +1640,7 @@ public class DataSetTableService { if (datasetTable.getMode() == 1 && StringUtils.equalsIgnoreCase("union", datasetTable.getType())) { datasetTableField.setDataeaseName(filed.getFieldName()); } else { - datasetTableField.setDataeaseName(DorisTableUtils.columnName(filed.getFieldName())); + datasetTableField.setDataeaseName(TableUtils.columnName(filed.getFieldName())); } datasetTableField.setType(filed.getFieldType()); if (ObjectUtils.isEmpty(ds)) { @@ -1665,7 +1668,7 @@ public class DataSetTableService { } private void createDorisView(String dorisTableName, String customSql) throws Exception { - Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource dorisDatasource = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(dorisDatasource); @@ -2165,12 +2168,12 @@ public class DataSetTableService { } public Boolean checkDorisTableIsExists(String id) throws Exception { - Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource dorisDatasource = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(dorisDatasource); QueryProvider qp = ProviderFactory.getQueryProvider(dorisDatasource.getType()); - datasourceRequest.setQuery(qp.searchTable(DorisTableUtils.dorisName(id))); + datasourceRequest.setQuery(qp.searchTable(TableUtils.tableName(id))); List data = jdbcProvider.getData(datasourceRequest); return CollectionUtils.isNotEmpty(data); } diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index ebf6e5bb9e..9795f86261 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -12,6 +12,7 @@ import io.dataease.commons.model.AuthURD; import io.dataease.commons.utils.*; import io.dataease.commons.constants.DatasourceTypes; import io.dataease.controller.request.datasource.ApiDefinition; +import io.dataease.provider.DDLProvider; import io.dataease.provider.datasource.DatasourceProvider; import io.dataease.provider.datasource.JdbcProvider; import io.dataease.provider.ProviderFactory; @@ -22,7 +23,8 @@ import io.dataease.dto.dataset.DataTableInfoDTO; import io.dataease.dto.dataset.ExcelSheetData; import io.dataease.exception.DataEaseException; import io.dataease.listener.util.CacheUtils; -import io.dataease.provider.query.QueryProvider; +import io.dataease.provider.QueryProvider; +import io.dataease.service.engine.EngineService; import io.dataease.service.message.DeMsgutil; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; @@ -94,6 +96,8 @@ public class ExtractDataService { private DatasourceService datasourceService; @Resource private ExtChartViewMapper extChartViewMapper; + @Resource + private EngineService engineService; private static final String lastUpdateTime = "${__last_update_time__}"; private static final String currentUpdateTime = "${__current_update_time__}"; @@ -111,13 +115,7 @@ public class ExtractDataService { private String user; @Value("${carte.passwd:cluster}") private String passwd; - private static final String creatTableSql = "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" + - "Column_Fields" + - "UNIQUE KEY(dataease_uuid)\n" + - "DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" + - "PROPERTIES(\"replication_num\" = \"1\");"; - private static final String dropTableSql = "DROP TABLE IF EXISTS TABLE_NAME;"; private static final String shellScript = "result=`curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load`\n" + "if [ $? -eq 0 ] ; then\n" + " failstatus=$(echo $result | grep '\"Status\": \"Fail\"')\n" + @@ -162,7 +160,7 @@ public class ExtractDataService { } UpdateType updateType = UpdateType.valueOf(type); DatasetTableTaskLog datasetTableTaskLog; - if(datasetTableFields == null){ + if (datasetTableFields == null) { datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build()); } datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList()); @@ -175,32 +173,36 @@ public class ExtractDataService { } return o1.getColumnIndex().compareTo(o2.getColumnIndex()); }); - String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields); + switch (updateType) { case all_scope: // 全量更新 try { datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops); - createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTableColumnSql); - createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTableColumnSql); - generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null); - generateJobFile("all_scope", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(","))); + createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields); + createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields); Long execTime = System.currentTimeMillis(); - extractData(datasetTable, "all_scope"); - replaceTable(DorisTableUtils.dorisName(datasetTableId)); + if (!engineService.isSimpleMode()) { + generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null); + generateJobFile("all_scope", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(","))); + extractData(datasetTable, "all_scope"); + } else { + extractExcelDataForSimpleMode(datasetTable, "all_scope"); + } + replaceTable(TableUtils.tableName(datasetTableId)); saveSuccessLog(datasetTableTaskLog); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); - if(ops.equalsIgnoreCase("替换")){ + if (ops.equalsIgnoreCase("替换")) { List oldFileds = getDatasetTableFields(datasetTable.getId()); List toAdd = new ArrayList<>(); List toDelete = new ArrayList<>(); for (DatasetTableField oldFiled : oldFileds) { boolean delete = true; for (DatasetTableField datasetTableField : datasetTableFields) { - if(oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())){ + if (oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())) { delete = false; } } - if(delete){ + if (delete) { toDelete.add(oldFiled); } } @@ -208,11 +210,11 @@ public class ExtractDataService { for (DatasetTableField datasetTableField : datasetTableFields) { boolean add = true; for (DatasetTableField oldFiled : oldFileds) { - if(oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())){ + if (oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())) { add = false; } } - if(add){ + if (add) { toAdd.add(datasetTableField); } } @@ -222,7 +224,7 @@ public class ExtractDataService { } catch (Exception e) { saveErrorLog(datasetTableId, null, e); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); - dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); + dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId))); } finally { deleteFile("all_scope", datasetTableId); deleteExcelFile(datasetTable, datasetTableIds); @@ -232,10 +234,14 @@ public class ExtractDataService { case add_scope: // 增量更新 try { datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops); - generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null); - generateJobFile("incremental_add", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(","))); Long execTime = System.currentTimeMillis(); - extractData(datasetTable, "incremental_add"); + if (!engineService.isSimpleMode()) { + generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null); + generateJobFile("incremental_add", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(","))); + extractData(datasetTable, "incremental_add"); + } else { + extractExcelDataForSimpleMode(datasetTable, "incremental_add"); + } saveSuccessLog(datasetTableTaskLog); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); } catch (Exception e) { @@ -257,21 +263,6 @@ public class ExtractDataService { } } - private List getDatasetTableFields(String datasetTableId){ - List datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build()); - datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList()); - datasetTableFields.sort((o1, o2) -> { - if (o1.getColumnIndex() == null) { - return -1; - } - if (o2.getColumnIndex() == null) { - return 1; - } - return o1.getColumnIndex().compareTo(o2.getColumnIndex()); - }); - return datasetTableFields; - } - public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) { DatasetTable datasetTable = getDatasetTable(datasetTableId); if (datasetTable == null) { @@ -306,19 +297,17 @@ public class ExtractDataService { datasource.setType(datasetTable.getType()); } List datasetTableFields = getDatasetTableFields(datasetTable.getId()); - String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields); - boolean msg = false; JobStatus lastExecStatus = JobStatus.Completed; Long execTime = null; switch (updateType) { case all_scope: // 全量更新 try { - createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTableColumnSql); - createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTableColumnSql); + createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields); + createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields); execTime = System.currentTimeMillis(); extractData(datasetTable, datasource, datasetTableFields, "all_scope", null); - replaceTable(DorisTableUtils.dorisName(datasetTableId)); + replaceTable(TableUtils.tableName(datasetTableId)); saveSuccessLog(datasetTableTaskLog); msg = true; lastExecStatus = JobStatus.Completed; @@ -328,21 +317,39 @@ public class ExtractDataService { lastExecStatus = JobStatus.Error; execTime = null; } finally { - try { deleteFile("all_scope", datasetTableId); }catch (Exception ignore){ System.out.println(ignore.getMessage());} - try { sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg); }catch (Exception ignore){ System.out.println(ignore.getMessage());} - try { dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus); }catch (Exception ignore){ + try { + deleteFile("all_scope", datasetTableId); + } catch (Exception ignore) { + System.out.println(ignore.getMessage()); + } + try { + sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg); + } catch (Exception ignore) { + System.out.println(ignore.getMessage()); + } + try { + dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus); + } catch (Exception ignore) { + System.out.println(ignore.getMessage()); + } + try { + updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime); + } catch (Exception ignore) { + System.out.println(ignore.getMessage()); + } + try { + dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId))); + } catch (Exception ignore) { System.out.println(ignore.getMessage()); } - try { updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime); }catch (Exception ignore){ System.out.println(ignore.getMessage());} - try { dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); }catch (Exception ignore){ System.out.println(ignore.getMessage());} } break; case add_scope: // 增量更新 try { - if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){ + if (datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())) { extractData(datasetTable, datasource, datasetTableFields, "incremental_add", null); - }else{ + } else { DatasetTableIncrementalConfig datasetTableIncrementalConfig = dataSetTableService.incrementalConfig(datasetTableId); if (datasetTable.getLastUpdateTime() == null || datasetTable.getLastUpdateTime() == 0) { throw new Exception("未进行全量同步"); @@ -370,10 +377,23 @@ public class ExtractDataService { lastExecStatus = JobStatus.Error; execTime = null; } finally { - try { deleteFile("incremental_add", datasetTableId); deleteFile("incremental_delete", datasetTableId); }catch (Exception ignore){} - try { sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg); }catch (Exception ignore){} - try { dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus); }catch (Exception ignore){} - try { updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime); }catch (Exception ignore){} + try { + deleteFile("incremental_add", datasetTableId); + deleteFile("incremental_delete", datasetTableId); + } catch (Exception ignore) { + } + try { + sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg); + } catch (Exception ignore) { + } + try { + dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus); + } catch (Exception ignore) { + } + try { + updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime); + } catch (Exception ignore) { + } } break; } @@ -385,43 +405,62 @@ public class ExtractDataService { } - private void extractData(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType, String selectSQL) throws Exception{ - if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){ - extractDataByDE(datasetTable, datasource, datasetTableFields, extractType); + private List getDatasetTableFields(String datasetTableId) { + List datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build()); + datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList()); + datasetTableFields.sort((o1, o2) -> { + if (o1.getColumnIndex() == null) { + return -1; + } + if (o2.getColumnIndex() == null) { + return 1; + } + return o1.getColumnIndex().compareTo(o2.getColumnIndex()); + }); + return datasetTableFields; + } + + private void extractData(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType, String selectSQL) throws Exception { + if (datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())) { + extractApiData(datasetTable, datasource, datasetTableFields, extractType); return; } extractDataByKettle(datasetTable, datasource, datasetTableFields, extractType, selectSQL); } - private void extractDataByDE(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType)throws Exception{ + private void extractApiData(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType) throws Exception { List lists = JSONObject.parseArray(datasource.getConfiguration(), ApiDefinition.class); lists = lists.stream().filter(item -> item.getName().equalsIgnoreCase(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable())).collect(Collectors.toList()); - if(CollectionUtils.isEmpty(lists)){ + if (CollectionUtils.isEmpty(lists)) { throw new Exception("未找到API数据表"); } - if(lists.size() > 1 ){ + if (lists.size() > 1) { throw new Exception("存在重名的API数据表"); } DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(datasource); datasourceRequest.setTable(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable()); - Map result = datasourceProvider.fetchResultAndField(datasourceRequest); + Map result = datasourceProvider.fetchResultAndField(datasourceRequest); List dataList = result.get("dataList"); - Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); - DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class); + if (engineService.isSimpleMode()) { + extractDataForSimpleMode(extractType, datasetTable.getId(), dataList); + } + + Datasource engine = engineService.getDeEngine(); + DorisConfiguration dorisConfiguration = new Gson().fromJson(engine.getConfiguration(), DorisConfiguration.class); String columns = datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")) + ",dataease_uuid"; String dataFile = null; String script = null; switch (extractType) { case "all_scope": - dataFile = root_path + DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention; - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId()))); + dataFile = root_path + TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())) + "." + extention; + script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()))); break; default: - dataFile = root_path + DorisTableUtils.dorisAddName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention; - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId())); + dataFile = root_path + TableUtils.addName(TableUtils.tableName(datasetTable.getId())) + "." + extention; + script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId())); break; } @@ -429,8 +468,8 @@ public class ExtractDataService { BufferedWriter bw = new BufferedWriter(new FileWriter(dataFile)); for (String[] strings : dataList) { String content = ""; - for (int i=0;i< strings.length;i++){ - content = content + strings[i] + separator; + for (int i = 0; i < strings.length; i++) { + content = content + strings[i] + separator; } content = content + Md5Utils.md5(content); bw.write(content); @@ -452,7 +491,7 @@ public class ExtractDataService { try { Process process = Runtime.getRuntime().exec(root_path + datasetTable.getId() + ".sh"); process.waitFor(); - if(process.waitFor() != 0){ + if (process.waitFor() != 0) { BufferedReader input = new BufferedReader(new InputStreamReader(process.getErrorStream())); String errMsg = ""; String line = ""; @@ -465,16 +504,45 @@ public class ExtractDataService { } throw new Exception(errMsg); } - }catch (Exception e){ + } catch (Exception e) { throw e; - }finally { + } finally { File deleteFile = new File(root_path + datasetTable.getId() + ".sh"); FileUtils.forceDelete(deleteFile); } } - private void extractDataByKettle(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType, String selectSQL)throws Exception{ + private void extractDataForSimpleMode(String extractType, String datasetId, List dataList) throws Exception { + String tableName; + switch (extractType) { + case "all_scope": + tableName = TableUtils.tmpName(TableUtils.tableName(datasetId)); + break; + default: + tableName = TableUtils.tableName(datasetId); + break; + } + Datasource engine = engineService.getDeEngine(); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(engine); + DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType()); + int pageNumber = 100; //一次插入 100条 + int totalPage; + if (dataList.size() % pageNumber > 0) { + totalPage = dataList.size() / pageNumber + 1; + } else { + totalPage = dataList.size() / pageNumber; + } + + for (int page = 1; page <= totalPage; page++) { + datasourceRequest.setQuery(ddlProvider.insertSql(tableName, dataList, page, pageNumber)); + jdbcProvider.exec(datasourceRequest); + } + } + + private void extractDataByKettle(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType, String selectSQL) throws Exception { generateTransFile(extractType, datasetTable, datasource, datasetTableFields, selectSQL); generateJobFile(extractType, datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(","))); extractData(datasetTable, extractType); @@ -542,70 +610,43 @@ public class ExtractDataService { } - private String createDorisTableColumnSql(final List datasetTableFields) { - StringBuilder Column_Fields = new StringBuilder("dataease_uuid varchar(50), `"); - for (DatasetTableField datasetTableField : datasetTableFields) { - Column_Fields.append(datasetTableField.getDataeaseName()).append("` "); - Integer size = datasetTableField.getSize() * 3; - if (datasetTableField.getSize() == 0 || datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) { - size = 65533; - } - switch (datasetTableField.getDeExtractType()) { - case 0: - Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`"); - break; - case 1: - size = size < 50? 50 : size; - Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`"); - break; - case 2: - Column_Fields.append("bigint").append(",`"); - break; - case 3: - Column_Fields.append("DOUBLE").append(",`"); - break; - case 4: - Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(size))).append(",`"); - break; - default: - Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`"); - break; - } - } - Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2)); - Column_Fields = new StringBuilder("(" + Column_Fields + ")\n"); - return Column_Fields.toString(); - } - - private void createDorisTable(String dorisTableName, String dorisTableColumnSql) throws Exception { - Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + private void createEngineTable(String tableName, List datasetTableFields) throws Exception { + Datasource engine = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); - datasourceRequest.setDatasource(dorisDatasource); - datasourceRequest.setQuery(creatTableSql.replace("TABLE_NAME", dorisTableName).replace("Column_Fields", dorisTableColumnSql)); + datasourceRequest.setDatasource(engine); + DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType()); + datasourceRequest.setQuery(ddlProvider.createTableSql(tableName, datasetTableFields)); jdbcProvider.exec(datasourceRequest); } - private void dropDorisTable(String dorisTableName) { + private void dropDorisTable(String tableName) { try { - Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource engine = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); - datasourceRequest.setDatasource(dorisDatasource); - datasourceRequest.setQuery(dropTableSql.replace("TABLE_NAME", dorisTableName)); + datasourceRequest.setDatasource(engine); + DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType()); + datasourceRequest.setQuery(ddlProvider.dropTable(tableName)); jdbcProvider.exec(datasourceRequest); } catch (Exception ignore) { } } private void replaceTable(String dorisTableName) throws Exception { - Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource engine = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); - datasourceRequest.setDatasource(dorisDatasource); - datasourceRequest.setQuery("ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false');".replace("DORIS_TABLE", dorisTableName).replace("DORIS_TMP_TABLE", DorisTableUtils.dorisTmpName(dorisTableName))); - jdbcProvider.exec(datasourceRequest); + datasourceRequest.setDatasource(engine); + DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType()); + String[] replaceTableSql = ddlProvider.replaceTable(dorisTableName).split(";"); + for (int i = 0; i < replaceTableSql.length; i++) { + if (StringUtils.isNotEmpty(replaceTableSql[i])) { + datasourceRequest.setQuery(replaceTableSql[i]); + jdbcProvider.exec(datasourceRequest); + } + } } private DatasetTable getDatasetTable(String datasetTableId) { @@ -661,8 +702,26 @@ public class ExtractDataService { return datasetTableTaskLog; } + private void extractExcelDataForSimpleMode(DatasetTable datasetTable, String extractType) throws Exception { + List data = new ArrayList<>(); + DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class); + List excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList(); + ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader(); + for (ExcelSheetData excelSheetData : excelSheetDataList) { + excelXlsxReader.process(new FileInputStream(excelSheetData.getPath())); + for (ExcelSheetData sheet : excelXlsxReader.totalSheets) { + if (sheet.getExcelLable().equalsIgnoreCase(excelSheetData.getExcelLable())) { + for (List dataItem : sheet.getData()) { + data.add(dataItem.toArray(new String[dataItem.size()])); + } + } + } + } + extractDataForSimpleMode(extractType, datasetTable.getId(), data); + } + private void extractData(DatasetTable datasetTable, String extractType) throws Exception { - if(StringUtils.isNotEmpty(datasetTable.getDataSourceId())){ + if (StringUtils.isNotEmpty(datasetTable.getDataSourceId())) { datasourceService.validate(datasetTable.getDataSourceId()); } KettleFileRepository repository = CommonBeanFactory.getBean(KettleFileRepository.class); @@ -671,16 +730,16 @@ public class ExtractDataService { JobMeta jobMeta = null; switch (extractType) { case "all_scope": - jobMeta = repository.loadJob("job_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null); - transMeta = repository.loadTransformation("trans_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, ""); + jobMeta = repository.loadJob("job_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null); + transMeta = repository.loadTransformation("trans_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, ""); break; case "incremental_add": - jobMeta = repository.loadJob("job_add_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null); - transMeta = repository.loadTransformation("trans_add_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, ""); + jobMeta = repository.loadJob("job_add_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null); + transMeta = repository.loadTransformation("trans_add_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, ""); break; case "incremental_delete": - jobMeta = repository.loadJob("job_delete_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null); - transMeta = repository.loadTransformation("trans_delete_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, ""); + jobMeta = repository.loadJob("job_delete_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null); + transMeta = repository.loadTransformation("trans_delete_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, ""); break; default: break; @@ -738,27 +797,30 @@ public class ExtractDataService { } private void generateJobFile(String extractType, DatasetTable datasetTable, String columnFields) throws Exception { + if (engineService.isSimpleMode()) { + return; + } String outFile; String jobName = null; String script = null; - Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource dorisDatasource = engineService.getDeEngine(); DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class); String columns = columnFields + ",dataease_uuid"; switch (extractType) { case "all_scope": - outFile = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())); - jobName = "job_" + DorisTableUtils.dorisName(datasetTable.getId()); - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())), root_path + outFile + "." + extention); + outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())); + jobName = "job_" + TableUtils.tableName(datasetTable.getId()); + script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention); break; case "incremental_add": - outFile = DorisTableUtils.dorisAddName(datasetTable.getId()); - jobName = "job_add_" + DorisTableUtils.dorisName(datasetTable.getId()); - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + outFile + "." + extention); + outFile = TableUtils.addName(datasetTable.getId()); + jobName = "job_add_" + TableUtils.tableName(datasetTable.getId()); + script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention); break; case "incremental_delete": - outFile = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(datasetTable.getId())); - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + outFile + "." + extention); - jobName = "job_delete_" + DorisTableUtils.dorisName(datasetTable.getId()); + outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId())); + script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention); + jobName = "job_delete_" + TableUtils.tableName(datasetTable.getId()); break; default: break; @@ -816,12 +878,15 @@ public class ExtractDataService { datasourceRequest.setQuery(qp.wrapSql(sql)); List dorisFields = new ArrayList<>(); datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName).forEach(filed -> { - dorisFields.add(DorisTableUtils.columnName(filed)); + dorisFields.add(TableUtils.columnName(filed)); }); return String.join(",", dorisFields); } private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String selectSQL) throws Exception { + if (engineService.isSimpleMode()) { + return; + } TransMeta transMeta = new TransMeta(); String outFile = null; DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasource.getType()); @@ -902,18 +967,18 @@ public class ExtractDataService { switch (extractType) { case "all_scope": - transName = "trans_" + DorisTableUtils.dorisName(datasetTable.getId()); - outFile = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())); + transName = "trans_" + TableUtils.tableName(datasetTable.getId()); + outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())); transMeta.setName(transName); break; case "incremental_add": - transName = "trans_add_" + DorisTableUtils.dorisName(datasetTable.getId()); - outFile = DorisTableUtils.dorisAddName(datasetTable.getId()); + transName = "trans_add_" + TableUtils.tableName(datasetTable.getId()); + outFile = TableUtils.addName(datasetTable.getId()); transMeta.setName(transName); break; case "incremental_delete": - transName = "trans_delete_" + DorisTableUtils.dorisName(datasetTable.getId()); - outFile = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(datasetTable.getId())); + transName = "trans_delete_" + TableUtils.tableName(datasetTable.getId()); + outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId())); transMeta.setName(transName); break; default: @@ -965,7 +1030,7 @@ public class ExtractDataService { return fromStep; } - private StepMeta excelInputStep(String Info, List datasetTableFields){ + private StepMeta excelInputStep(String Info, List datasetTableFields) { DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class); List excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList(); String suffix = excelSheetDataList.get(0).getPath().substring(excelSheetDataList.get(0).getPath().lastIndexOf(".") + 1); @@ -975,10 +1040,10 @@ public class ExtractDataService { List files = new ArrayList<>(); List filesRequired = new ArrayList<>(); for (ExcelSheetData excelSheetData : excelSheetDataList) { - if(!sheetNames.contains(excelSheetData.getExcelLable())){ + if (!sheetNames.contains(excelSheetData.getExcelLable())) { sheetNames.add(excelSheetData.getExcelLable()); } - if(!files.contains(excelSheetData.getPath())){ + if (!files.contains(excelSheetData.getPath())) { files.add(excelSheetData.getPath()); filesRequired.add("Y"); } @@ -992,7 +1057,7 @@ public class ExtractDataService { excelInputMeta.setSheetName(sheetNames.toArray(new String[sheetNames.size()])); } excelInputMeta.setPassword("Encrypted"); - excelInputMeta.setFileName( files.toArray(new String[files.size()])); + excelInputMeta.setFileName(files.toArray(new String[files.size()])); excelInputMeta.setFileRequired(filesRequired.toArray(new String[filesRequired.size()])); excelInputMeta.setStartsWithHeader(true); excelInputMeta.setIgnoreEmptyRows(true); @@ -1024,9 +1089,9 @@ public class ExtractDataService { textFileOutputMeta.setSeparator(separator); textFileOutputMeta.setExtension(extention); - if (datasource.getType().equalsIgnoreCase(DatasourceTypes.oracle.name()) ) { + if (datasource.getType().equalsIgnoreCase(DatasourceTypes.oracle.name())) { TextFileField[] outputFields = new TextFileField[datasetTableFields.size() + 1]; - for(int i=0;i< datasetTableFields.size();i++){ + for (int i = 0; i < datasetTableFields.size(); i++) { TextFileField textFileField = new TextFileField(); textFileField.setName(datasetTableFields.get(i).getOriginName()); textFileField.setType("String"); @@ -1038,9 +1103,9 @@ public class ExtractDataService { outputFields[datasetTableFields.size()] = textFileField; textFileOutputMeta.setOutputFields(outputFields); - }else if (datasource.getType().equalsIgnoreCase(DatasourceTypes.sqlServer.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.pg.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.mysql.name())){ + } else if (datasource.getType().equalsIgnoreCase(DatasourceTypes.sqlServer.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.pg.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.mysql.name())) { TextFileField[] outputFields = new TextFileField[datasetTableFields.size() + 1]; - for(int i=0;i< datasetTableFields.size();i++){ + for (int i = 0; i < datasetTableFields.size(); i++) { TextFileField textFileField = new TextFileField(); textFileField.setName(datasetTableFields.get(i).getDataeaseName()); if (datasetTableFields.get(i).getDeExtractType().equals(DeTypeConstants.DE_TIME)) { @@ -1058,9 +1123,9 @@ public class ExtractDataService { outputFields[datasetTableFields.size()] = textFileField; textFileOutputMeta.setOutputFields(outputFields); - }else if(datasource.getType().equalsIgnoreCase(DatasourceTypes.excel.name())) { + } else if (datasource.getType().equalsIgnoreCase(DatasourceTypes.excel.name())) { TextFileField[] outputFields = new TextFileField[datasetTableFields.size() + 1]; - for(int i=0;i< datasetTableFields.size();i++){ + for (int i = 0; i < datasetTableFields.size(); i++) { TextFileField textFileField = new TextFileField(); textFileField.setName(datasetTableFields.get(i).getDataeaseName()); if (datasetTableFields.get(i).getDeExtractType().equals(DeTypeConstants.DE_INT)) { @@ -1078,7 +1143,7 @@ public class ExtractDataService { outputFields[datasetTableFields.size()] = textFileField; textFileOutputMeta.setOutputFields(outputFields); - }else { + } else { textFileOutputMeta.setOutputFields(new TextFileField[0]); } @@ -1093,7 +1158,7 @@ public class ExtractDataService { String excelCompletion = ""; for (DatasetTableField datasetTableField : datasetTableFields) { - if(datasetTableField.getDeExtractType().equals(DeTypeConstants.DE_BINARY)){ + if (datasetTableField.getDeExtractType().equals(DeTypeConstants.DE_BINARY)) { handleBinaryTypeCode.append("\n").append(handleBinaryType.replace("FIELD", datasetTableField.getDataeaseName())); } } @@ -1134,7 +1199,7 @@ public class ExtractDataService { } public void deleteFile(String type, String dataSetTableId) { - if(kettleFilesKeep){ + if (kettleFilesKeep) { return; } String transName = null; @@ -1143,19 +1208,19 @@ public class ExtractDataService { switch (type) { case "all_scope": - transName = "trans_" + DorisTableUtils.dorisName(dataSetTableId); - jobName = "job_" + DorisTableUtils.dorisName(dataSetTableId); - fileName = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(dataSetTableId)); + transName = "trans_" + TableUtils.tableName(dataSetTableId); + jobName = "job_" + TableUtils.tableName(dataSetTableId); + fileName = TableUtils.tmpName(TableUtils.tableName(dataSetTableId)); break; case "incremental_add": - transName = "trans_add_" + DorisTableUtils.dorisName(dataSetTableId); - jobName = "job_add_" + DorisTableUtils.dorisName(dataSetTableId); - fileName = DorisTableUtils.dorisAddName(DorisTableUtils.dorisName(dataSetTableId)); + transName = "trans_add_" + TableUtils.tableName(dataSetTableId); + jobName = "job_add_" + TableUtils.tableName(dataSetTableId); + fileName = TableUtils.addName(TableUtils.tableName(dataSetTableId)); break; case "incremental_delete": - transName = "trans_delete_" + DorisTableUtils.dorisName(dataSetTableId); - jobName = "job_delete_" + DorisTableUtils.dorisName(dataSetTableId); - fileName = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(dataSetTableId)); + transName = "trans_delete_" + TableUtils.tableName(dataSetTableId); + jobName = "job_delete_" + TableUtils.tableName(dataSetTableId); + fileName = TableUtils.deleteName(TableUtils.tableName(dataSetTableId)); break; default: break; @@ -1165,29 +1230,33 @@ public class ExtractDataService { deleteFile(root_path + transName + ".ktr"); } - private void deleteExcelFile(DatasetTable datasetTable, ListdatasetTableIds){ + private void deleteExcelFile(DatasetTable datasetTable, List datasetTableIds) { List datasetTables = dataSetTableService.list(datasetTableIds); for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) { Boolean allIsFinished = true; for (DatasetTable table : datasetTables) { - for(ExcelSheetData data : new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()){ - if(data.getPath().equalsIgnoreCase(excelSheetData.getPath())){ - if(StringUtils.isEmpty(table.getSyncStatus()) || table.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())){ + for (ExcelSheetData data : new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) { + if (data.getPath().equalsIgnoreCase(excelSheetData.getPath())) { + if (StringUtils.isEmpty(table.getSyncStatus()) || table.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())) { allIsFinished = false; } } } } - if(allIsFinished){ + if (allIsFinished) { deleteFile(excelSheetData.getPath()); - }else { - try { Thread.sleep(5000); }catch (Exception ignore){} + } else { + try { + Thread.sleep(5000); + } catch (Exception ignore) { + } deleteExcelFile(datasetTable, datasetTableIds); } } } - private void deleteFile(String filePath){ - if(StringUtils.isEmpty(filePath)){ + + private void deleteFile(String filePath) { + if (StringUtils.isEmpty(filePath)) { return; } try { @@ -1196,6 +1265,7 @@ public class ExtractDataService { } catch (Exception e) { } } + public boolean isKettleRunning() { try { if (!InetAddress.getByName(carte).isReachable(1000)) { diff --git a/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java b/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java index ea36b7d970..20c54d4786 100644 --- a/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java +++ b/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java @@ -5,7 +5,6 @@ import io.dataease.base.domain.DatasetTable; import io.dataease.base.domain.DatasetTableField; import io.dataease.base.domain.Datasource; import io.dataease.commons.constants.ColumnPermissionConstants; -import io.dataease.commons.utils.CommonBeanFactory; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.i18n.Translator; import io.dataease.provider.datasource.DatasourceProvider; @@ -15,7 +14,8 @@ import io.dataease.service.dataset.*; import io.dataease.service.datasource.DatasourceService; import io.dataease.dto.dataset.DataSetTableUnionDTO; import io.dataease.dto.dataset.DataTableInfoDTO; -import io.dataease.provider.query.QueryProvider; +import io.dataease.provider.QueryProvider; +import io.dataease.service.engine.EngineService; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -40,6 +40,8 @@ public class DirectFieldService implements DataSetFieldService { private DataSetTableUnionService dataSetTableUnionService; @Resource private PermissionService permissionService; + @Resource + private EngineService engineService; @Override public List fieldValues(String fieldId, Long userId, Boolean userPermissions) throws Exception { @@ -100,7 +102,7 @@ public class DirectFieldService implements DataSetFieldService { } } else if (datasetTable.getMode() == 1) {// 抽取 // 连接doris,构建doris数据源查询 - Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + Datasource ds = engineService.getDeEngine(); datasourceProvider = ProviderFactory.getProvider(ds.getType()); datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); diff --git a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java index 4d2fecdaf5..0f3b597bcc 100644 --- a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java +++ b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java @@ -71,7 +71,7 @@ public class DatasourceService { return datasource; } - private void handleConnectionPool(Datasource datasource, String type) { + public void handleConnectionPool(Datasource datasource, String type) { commonThreadPool.addTask(() -> { try { DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); @@ -94,7 +94,7 @@ public class DatasourceService { switch (datasourceType) { case mysql: case mariadb: - case de_doris: + case engine_doris: case ds_doris: datasourceDTO.setConfiguration(JSONObject.toJSONString(new Gson().fromJson(datasourceDTO.getConfiguration(), MysqlConfiguration.class)) ); break; @@ -221,7 +221,6 @@ public class DatasourceService { }catch (Exception e){ return ResultHolder.error("Datasource is invalid: " + e.getMessage()); } - } public ResultHolder validate(String datasourceId) { diff --git a/backend/src/main/java/io/dataease/service/engine/EngineService.java b/backend/src/main/java/io/dataease/service/engine/EngineService.java new file mode 100644 index 0000000000..627c67e311 --- /dev/null +++ b/backend/src/main/java/io/dataease/service/engine/EngineService.java @@ -0,0 +1,121 @@ +package io.dataease.service.engine; + +import com.alibaba.fastjson.JSONObject; +import io.dataease.base.domain.Datasource; +import io.dataease.base.domain.DeEngine; +import io.dataease.base.domain.DeEngineExample; +import io.dataease.base.mapper.DeEngineMapper; +import io.dataease.commons.utils.BeanUtils; +import io.dataease.controller.ResultHolder; +import io.dataease.controller.request.datasource.DatasourceRequest; +import io.dataease.dto.DatasourceDTO; +import io.dataease.provider.ProviderFactory; +import io.dataease.provider.datasource.DatasourceProvider; +import io.dataease.service.datasource.DatasourceService; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.util.List; +import java.util.UUID; + +@Service +@Transactional(rollbackFor = Exception.class) +public class EngineService { + @Resource + private Environment env; + @Resource + private DeEngineMapper deEngineMapper; + @Resource + private DatasourceService datasource; + static private Datasource ds = new Datasource(); + + + public Boolean isLocalMode(){ + return env.getProperty("engine_mode", "local").equalsIgnoreCase("local"); + } + + public Boolean isSimpleMode(){ + return env.getProperty("engine_mode", "local").equalsIgnoreCase("simple"); + } + + public Boolean isClusterMode(){ + return env.getProperty("engine_mode", "local").equalsIgnoreCase("cluster"); + } + + public String mode(){ + return env.getProperty("engine_mode", "local"); + } + + public DeEngine info(){ + List deEngines = deEngineMapper.selectByExampleWithBLOBs(new DeEngineExample()); + if(CollectionUtils.isEmpty(deEngines)){ + return new DeEngine(); + } + return deEngines.get(0); + } + + public ResultHolder validate(DatasourceDTO datasource) throws Exception { + try { + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(datasource); + datasourceProvider.checkStatus(datasourceRequest); + return ResultHolder.success(datasource); + }catch (Exception e){ + return ResultHolder.error("Datasource is invalid: " + e.getMessage()); + } + } + + public ResultHolder save(DeEngine engine) throws Exception { + if(StringUtils.isEmpty(engine.getId())){ + engine.setId(UUID.randomUUID().toString()); + deEngineMapper.insert(engine); + }else { + deEngineMapper.updateByPrimaryKeyWithBLOBs(engine); + } + datasource.handleConnectionPool(this.ds, "delete"); + BeanUtils.copyBean(this.ds, engine); + datasource.handleConnectionPool(this.ds, "add"); + return ResultHolder.success(engine); + } + + public Datasource getDeEngine() throws Exception{ + if (this.ds != null || StringUtils.isNotEmpty(ds.getType())) { + return this.ds; + } + if(isLocalMode()){ + JSONObject jsonObject = new JSONObject(); + jsonObject.put("dataSourceType", "jdbc"); + jsonObject.put("dataBase", env.getProperty("doris.db", "doris")); + jsonObject.put("username", env.getProperty("doris.user", "root")); + jsonObject.put("password", env.getProperty("doris.password", "dataease")); + jsonObject.put("host", env.getProperty("doris.host", "doris")); + jsonObject.put("port", env.getProperty("doris.port", "9030")); + jsonObject.put("httpPort", env.getProperty("doris.httpPort", "8030")); + + Datasource datasource = new Datasource(); + datasource.setId("doris"); + datasource.setName("doris"); + datasource.setDesc("doris"); + datasource.setType("engine_doris"); + datasource.setConfiguration(jsonObject.toJSONString()); + this.ds = datasource; + } + if(isSimpleMode()){ + List deEngines = deEngineMapper.selectByExampleWithBLOBs(new DeEngineExample()); + if(CollectionUtils.isEmpty(deEngines)){ + throw new Exception("未设置数据引擎"); + } + BeanUtils.copyBean(this.ds, deEngines.get(0)); + } + + //TODO cluster mode + return this.ds; + } + + +} diff --git a/backend/src/main/resources/db/migration/V33__1.9.sql b/backend/src/main/resources/db/migration/V33__1.9.sql index f08e9d9d1a..306221699a 100644 --- a/backend/src/main/resources/db/migration/V33__1.9.sql +++ b/backend/src/main/resources/db/migration/V33__1.9.sql @@ -68,3 +68,16 @@ CREATE TABLE `panel_group_extend_data` ( SET FOREIGN_KEY_CHECKS = 1; + +CREATE TABLE `de_engine` ( + `id` varchar(50) NOT NULL DEFAULT '' COMMENT 'ID', + `name` varchar(50) DEFAULT NULL COMMENT '名称', + `desc` varchar(50) DEFAULT NULL COMMENT '描述', + `type` varchar(50) NOT NULL COMMENT '类型', + `configuration` longtext NOT NULL COMMENT '详细信息', + `create_time` bigint(13) DEFAULT NULL COMMENT 'Create timestamp', + `update_time` bigint(13) DEFAULT NULL COMMENT 'Update timestamp', + `create_by` varchar(50) DEFAULT NULL COMMENT '创建人ID', + `status` varchar(45) DEFAULT NULL COMMENT '状态', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/backend/src/main/resources/generatorConfig.xml b/backend/src/main/resources/generatorConfig.xml index f0ffa9f387..d5256f1916 100644 --- a/backend/src/main/resources/generatorConfig.xml +++ b/backend/src/main/resources/generatorConfig.xml @@ -60,17 +60,9 @@ - -
- - - - -
- - - - - +
+ +
+ diff --git a/frontend/src/api/system/engine.js b/frontend/src/api/system/engine.js new file mode 100644 index 0000000000..a0dd8c9a4d --- /dev/null +++ b/frontend/src/api/system/engine.js @@ -0,0 +1,36 @@ +import request from '@/utils/request' +import {validateDs} from "@/api/system/datasource"; + +export function engineMode() { + return request({ + url: '/engine/mode', + method: 'get', + loading: true + }) +} + +export function engineInfo() { + return request({ + url: '/engine/info', + method: 'get', + loading: true + }) +} + +export function validate(data) { + return request({ + url: '/engine/validate', + method: 'post', + loading: true, + data + }) +} + +export function save(data) { + return request({ + url: '/engine/save', + method: 'post', + loading: true, + data + }) +} diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index b01fc62e48..7c403d74cd 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -647,7 +647,8 @@ export default { port: 'Port number cannot be empty', account: 'Account cannot be empty', test_recipients: 'Test recipients', - tip: 'Tip: use as test mail recipient only' + tip: 'Tip: use as test mail recipient only', + engine_mode_setting: 'Engine Setting' }, chart: { save_snapshot: 'Save Snapshot', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index edd2787ce9..733a11340a 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -649,7 +649,8 @@ export default { port: '端口號不能爲空', account: '賬戶不能爲空', test_recipients: '測試收件人', - tip: '提示:僅用來作爲測試郵件收件人' + tip: '提示:僅用來作爲測試郵件收件人', + engine_mode_setting: '引擎設置' }, chart: { save_snapshot: '保存縮略圖', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 186cb46870..d0cbf18779 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -650,7 +650,8 @@ export default { port: '端口号不能为空', account: '账户不能为空', test_recipients: '测试收件人', - tip: '提示:仅用来作为测试邮件收件人' + tip: '提示:仅用来作为测试邮件收件人', + engine_mode_setting: '引擎设置' }, chart: { save_snapshot: '保存缩略图', diff --git a/frontend/src/views/dataset/add/AddApi.vue b/frontend/src/views/dataset/add/AddApi.vue index f15ea8f66c..ca72c61192 100644 --- a/frontend/src/views/dataset/add/AddApi.vue +++ b/frontend/src/views/dataset/add/AddApi.vue @@ -28,7 +28,7 @@ - + @@ -66,6 +66,7 @@ diff --git a/frontend/src/views/dataset/add/AddSQL.vue b/frontend/src/views/dataset/add/AddSQL.vue index 0757a854f4..c778f7c217 100644 --- a/frontend/src/views/dataset/add/AddSQL.vue +++ b/frontend/src/views/dataset/add/AddSQL.vue @@ -35,7 +35,7 @@ + :disabled="disabledSync"/> @@ -124,6 +124,7 @@ import 'codemirror/keymap/emacs.js' import 'codemirror/addon/hint/show-hint.css' import 'codemirror/addon/hint/sql-hint' import 'codemirror/addon/hint/show-hint' +import {engineMode} from "@/api/system/engine"; export default { name: 'AddSQL', @@ -157,7 +158,9 @@ export default { syncType: 'sync_now', height: 500, kettleRunning: false, - selectedDatasource: {} + selectedDatasource: {}, + engineMode: 'local', + disabledSync: true } }, computed: { @@ -187,6 +190,9 @@ export default { }, created() { this.kettleState() + engineMode().then(res => { + this.engineMode = res.data + }) }, methods: { kettleState() { @@ -198,6 +204,11 @@ export default { for (let i = 0; i < this.options.length; i++) { if (this.options[i].id === this.dataSource) { this.selectedDatasource = this.options[i] + if (this.engineMode === 'simple' || (!this.kettleRunning || this.disabledSyncDs.indexOf(this.selectedDatasource.type) !== -1 )) { + this.disabledSync = true + } else { + this.disabledSync = false + } } } }, diff --git a/frontend/src/views/dataset/common/DatasetGroupSelectorTree.vue b/frontend/src/views/dataset/common/DatasetGroupSelectorTree.vue index c0feb93984..d3308be7b9 100644 --- a/frontend/src/views/dataset/common/DatasetGroupSelectorTree.vue +++ b/frontend/src/views/dataset/common/DatasetGroupSelectorTree.vue @@ -79,7 +79,6 @@ + + diff --git a/frontend/src/views/system/SysParam/index.vue b/frontend/src/views/system/SysParam/index.vue index 400c32d411..a3df8dc9d6 100644 --- a/frontend/src/views/system/SysParam/index.vue +++ b/frontend/src/views/system/SysParam/index.vue @@ -26,32 +26,42 @@ + + + +