From 904d2c95a26646484a62d49bc92171fc8b1add84 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Thu, 1 Dec 2022 17:13:21 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(=E5=AE=9A=E6=97=B6=E6=8A=A5=E5=91=8A):?= =?UTF-8?q?=20=E4=B8=8B=E8=BD=BD=E5=AE=9A=E6=97=B6=E6=8A=A5=E5=91=8A?= =?UTF-8?q?=E5=9B=BE=E7=89=87api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../auth/annotation/DeRateLimiter.java | 24 +++++++ .../auth/aop/DeRateLimiterHandler.java | 55 +++++++++++++++ .../dataease/auth/service/DeLimitService.java | 8 +++ .../service/impl/RedisLimitServiceImpl.java | 51 ++++++++++++++ .../impl/StandaloneLimitServiceImpl.java | 26 +++++++ .../java/io/dataease/config/RedisConfig.java | 12 ++++ .../plugins/server/XEmailTaskServer.java | 70 ++++++++++++++++--- backend/src/main/resources/scripts/limit.lua | 27 +++++++ 8 files changed, 263 insertions(+), 10 deletions(-) create mode 100644 backend/src/main/java/io/dataease/auth/annotation/DeRateLimiter.java create mode 100644 backend/src/main/java/io/dataease/auth/aop/DeRateLimiterHandler.java create mode 100644 backend/src/main/java/io/dataease/auth/service/DeLimitService.java create mode 100644 backend/src/main/java/io/dataease/auth/service/impl/RedisLimitServiceImpl.java create mode 100644 backend/src/main/java/io/dataease/auth/service/impl/StandaloneLimitServiceImpl.java create mode 100644 backend/src/main/resources/scripts/limit.lua diff --git a/backend/src/main/java/io/dataease/auth/annotation/DeRateLimiter.java b/backend/src/main/java/io/dataease/auth/annotation/DeRateLimiter.java new file mode 100644 index 0000000000..928f55349e --- /dev/null +++ b/backend/src/main/java/io/dataease/auth/annotation/DeRateLimiter.java @@ -0,0 +1,24 @@ +package io.dataease.auth.annotation; + +import org.springframework.core.annotation.AliasFor; + +import java.lang.annotation.*; +import java.util.concurrent.TimeUnit; + +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface DeRateLimiter { + + long DEFAULT_REQUEST = 2; + + @AliasFor("max") long value() default DEFAULT_REQUEST; + + @AliasFor("value") long max() default DEFAULT_REQUEST; + + String key() default ""; + + long timeout() default 500; + + TimeUnit timeUnit() default TimeUnit.MILLISECONDS; +} diff --git a/backend/src/main/java/io/dataease/auth/aop/DeRateLimiterHandler.java b/backend/src/main/java/io/dataease/auth/aop/DeRateLimiterHandler.java new file mode 100644 index 0000000000..8bd16ddae3 --- /dev/null +++ b/backend/src/main/java/io/dataease/auth/aop/DeRateLimiterHandler.java @@ -0,0 +1,55 @@ +package io.dataease.auth.aop; + +import cn.hutool.core.util.StrUtil; +import io.dataease.auth.annotation.DeRateLimiter; +import io.dataease.auth.service.DeLimitService; +import io.dataease.commons.utils.IPUtils; +import io.dataease.commons.utils.ServletUtils; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.reflect.MethodSignature; +import org.springframework.core.annotation.AnnotationUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.lang.reflect.Method; +import java.util.concurrent.TimeUnit; + +@Aspect +@Component +public class DeRateLimiterHandler { + + private final static String SEPARATOR = ":"; + + + @Resource + private DeLimitService deLimitService; + + + @Around(value = "@annotation(io.dataease.auth.annotation.DeRateLimiter)") + public Object around(ProceedingJoinPoint point) throws Throwable { + MethodSignature signature = (MethodSignature) point.getSignature(); + Method method = signature.getMethod(); + DeRateLimiter rateLimiter = AnnotationUtils.findAnnotation(method, DeRateLimiter.class); + if (rateLimiter != null) { + String key = rateLimiter.key(); + if (StrUtil.isBlank(key)) { + key = method.getDeclaringClass().getName() + StrUtil.DOT + method.getName(); + } + key = key + SEPARATOR + IPUtils.get(); + + long max = rateLimiter.max(); + long timeout = rateLimiter.timeout(); + TimeUnit timeUnit = rateLimiter.timeUnit(); + Boolean limited = deLimitService.checkRestricted(key, max, timeout, timeUnit); + if (limited) { + String msg = "The current API [%s] is limited, please try again later!"; + String requestURI = ServletUtils.request().getRequestURI(); + throw new RuntimeException(String.format(msg, requestURI)); + } + } + + return point.proceed(); + } +} diff --git a/backend/src/main/java/io/dataease/auth/service/DeLimitService.java b/backend/src/main/java/io/dataease/auth/service/DeLimitService.java new file mode 100644 index 0000000000..79c4b6c9b8 --- /dev/null +++ b/backend/src/main/java/io/dataease/auth/service/DeLimitService.java @@ -0,0 +1,8 @@ +package io.dataease.auth.service; + +import java.util.concurrent.TimeUnit; + +public interface DeLimitService { + + Boolean checkRestricted(String key, long max, long timeout, TimeUnit timeUnit); +} diff --git a/backend/src/main/java/io/dataease/auth/service/impl/RedisLimitServiceImpl.java b/backend/src/main/java/io/dataease/auth/service/impl/RedisLimitServiceImpl.java new file mode 100644 index 0000000000..b066f951b5 --- /dev/null +++ b/backend/src/main/java/io/dataease/auth/service/impl/RedisLimitServiceImpl.java @@ -0,0 +1,51 @@ +package io.dataease.auth.service.impl; + +import io.dataease.auth.service.DeLimitService; +import io.dataease.commons.condition.RedisStatusCondition; +import io.dataease.commons.utils.LogUtil; +import org.slf4j.Logger; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Primary; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.RedisScript; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.time.Instant; +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +@Conditional({RedisStatusCondition.class}) +@Component +@Primary +public class RedisLimitServiceImpl implements DeLimitService { + + Logger log = LogUtil.getLogger(); + private final static String REDIS_LIMIT_KEY_PREFIX = "limit:"; + @Resource + private RedisScript limitRedisScript; + + @Resource + private StringRedisTemplate stringRedisTemplate; + + @Override + public Boolean checkRestricted(String key, long max, long timeout, TimeUnit timeUnit) { + key = REDIS_LIMIT_KEY_PREFIX + key; + long ttl = timeUnit.toMillis(timeout); + long now = Instant.now().toEpochMilli(); + long expired = now - ttl; + + Long executeTimes = stringRedisTemplate.execute(limitRedisScript, Collections.singletonList(key), now + "", ttl + "", expired + "", max + ""); + if (executeTimes != null) { + if (executeTimes == 0) { + + log.error("【{}】在单位时间 {} 毫秒内已达到访问上限，当前接口上限 {}", key, ttl, max); + return true; + } else { + log.info("【{}】在单位时间 {} 毫秒内访问 {} 次", key, ttl, executeTimes); + return false; + } + } + return false; + } +} diff --git a/backend/src/main/java/io/dataease/auth/service/impl/StandaloneLimitServiceImpl.java b/backend/src/main/java/io/dataease/auth/service/impl/StandaloneLimitServiceImpl.java new file mode 100644 index 0000000000..3888b680ea --- /dev/null +++ b/backend/src/main/java/io/dataease/auth/service/impl/StandaloneLimitServiceImpl.java @@ -0,0 +1,26 @@ +package io.dataease.auth.service.impl; + + +import com.google.common.util.concurrent.RateLimiter; +import io.dataease.auth.service.DeLimitService; +import org.springframework.stereotype.Service; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + + +@Service +public class StandaloneLimitServiceImpl implements DeLimitService { + + private static ConcurrentHashMap RATE_LIMITER = new ConcurrentHashMap<>(); + + @Override + public Boolean checkRestricted(String key, long max, long timeout, TimeUnit timeUnit) { + RateLimiter rateLimiter = null; + if (!RATE_LIMITER.containsKey(key)) { + RATE_LIMITER.put(key, RateLimiter.create(max)); + } + rateLimiter = RATE_LIMITER.get(key); + return !rateLimiter.tryAcquire(timeout, timeUnit); + } +} diff --git a/backend/src/main/java/io/dataease/config/RedisConfig.java b/backend/src/main/java/io/dataease/config/RedisConfig.java index 765d6014c4..66ac83b55c 100644 --- a/backend/src/main/java/io/dataease/config/RedisConfig.java +++ b/backend/src/main/java/io/dataease/config/RedisConfig.java @@ -6,10 +6,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.script.DefaultRedisScript; +import org.springframework.data.redis.core.script.RedisScript; import org.springframework.data.redis.listener.RedisMessageListenerContainer; import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; +import org.springframework.scripting.support.ResourceScriptSource; @Conditional({RedisStatusCondition.class}) @@ -36,4 +40,12 @@ public class RedisConfig { return container; } + @Bean + public RedisScript limitRedisScript() { + DefaultRedisScript redisScript = new DefaultRedisScript<>(); + redisScript.setScriptSource(new ResourceScriptSource(new ClassPathResource("scripts/limit.lua"))); + redisScript.setResultType(Long.class); + return redisScript; + } + } diff --git a/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java b/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java index ecd35e8214..af2b788f71 100644 --- a/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java +++ b/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java @@ -1,8 +1,10 @@ package io.dataease.plugins.server; import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.ArrayUtil; import com.github.pagehelper.Page; import com.github.pagehelper.PageHelper; +import io.dataease.auth.annotation.DeRateLimiter; import io.dataease.auth.api.dto.CurrentUserDto; import io.dataease.commons.exception.DEException; import io.dataease.commons.model.excel.ExcelSheetModel; @@ -27,6 +29,9 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.shiro.authz.annotation.RequiresPermissions; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.*; +import org.springframework.util.Base64Utils; import org.springframework.web.bind.annotation.*; import org.springframework.web.util.HtmlUtils; import springfox.documentation.annotations.ApiIgnore; @@ -150,7 +155,51 @@ public class XEmailTaskServer { return xpackEmailCreate; } - @PostMapping("/preview") + @DeRateLimiter + @GetMapping("/testApple") + public String testApple() { + return "调用api成功"; + } + + @DeRateLimiter + @PostMapping(value = "/screenshot", produces = {MediaType.IMAGE_JPEG_VALUE, MediaType.IMAGE_PNG_VALUE}) + public ResponseEntity screenshot(@RequestBody XpackEmailViewRequest request) { + EmailXpackService emailXpackService = SpringContextUtil.getBean(EmailXpackService.class); + String url = ServletUtils.domain() + "/#/previewScreenShot/" + request.getPanelId() + "/true"; + byte[] bytes = null; + try { + String currentToken = ServletUtils.getToken(); + Future future = priorityExecutor.submit(() -> { + try { + return emailXpackService.print(url, currentToken, buildPixel(request.getPixel())); + } catch (Exception e) { + LogUtil.error(e.getMessage(), e); + DEException.throwException("预览失败，请联系管理员"); + } + return null; + }, 0); + Object object = future.get(); + if (ObjectUtils.isNotEmpty(object)) { + bytes = (byte[]) object; + if (ArrayUtil.isNotEmpty(bytes)) { + String fileName = request.getPanelId() + ".jpeg"; + ByteArrayResource bar = new ByteArrayResource(bytes); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); + ContentDisposition contentDisposition = ContentDisposition.parse("attachment; filename=" + URLEncoder.encode(fileName, "UTF-8")); + headers.setContentDisposition(contentDisposition); + return new ResponseEntity(bar, headers, HttpStatus.OK); + } + } + } catch (Exception e) { + LogUtil.error(e.getMessage(), e); + DEException.throwException("预览失败，请联系管理员"); + } + + return null; + } + + @PostMapping(value = "/preview") public String preview(@RequestBody XpackEmailViewRequest request) { EmailXpackService emailXpackService = SpringContextUtil.getBean(EmailXpackService.class); String panelId = request.getPanelId(); @@ -159,7 +208,6 @@ public class XEmailTaskServer { String url = ServletUtils.domain() + "/#/previewScreenShot/" + panelId + "/true"; String token = ServletUtils.getToken(); - String fileId = null; try { Future future = priorityExecutor.submit(() -> { try { @@ -172,19 +220,21 @@ public class XEmailTaskServer { }, 0); Object object = future.get(); if (ObjectUtils.isNotEmpty(object)) { - fileId = object.toString(); + byte[] bytes = (byte[]) object; + String baseCode = Base64Utils.encodeToString(bytes); + String imageUrl = "data:image/jpeg;base64," + baseCode; + String html = "
" + + content + + "" + + "
"; + + return html; } } catch (Exception e) { LogUtil.error(e.getMessage(), e); DEException.throwException("预览失败，请联系管理员"); } - String imageUrl = "/system/ui/image/" + fileId; - String html = "
" + - content + - "" + - "
"; - - return html; + return null; } diff --git a/backend/src/main/resources/scripts/limit.lua b/backend/src/main/resources/scripts/limit.lua new file mode 100644 index 0000000000..43d5db7524 --- /dev/null +++ b/backend/src/main/resources/scripts/limit.lua @@ -0,0 +1,27 @@ +-- 下标从 1 开始 获取key +local key = KEYS[1] +-- 下标从 1 开始 获取参数 +local now = tonumber(ARGV[1]) -- 当前时间错 +local ttl = tonumber(ARGV[2]) -- 有效 +local expired = tonumber(ARGV[3]) -- +local max = tonumber(ARGV[4]) + +-- 清除过期的数据 +-- 移除指定分数区间内的所有元素，expired 即已经过期的 score +-- 根据当前时间毫秒数 - 超时毫秒数，得到过期时间 expired +redis.call('zremrangebyscore', key, 0, expired) + +-- 获取 zset 中的当前元素个数 +local current = tonumber(redis.call('zcard', key)) +local next = current + 1 + +if next > max then + -- 达到限流大小 返回 0 + return 0; +else + -- 往 zset 中添加一个值、得分均为当前时间戳的元素，[value,score] + redis.call("zadd", key, now, now) + -- 每次访问均重新设置 zset 的过期时间，单位毫秒 + redis.call("pexpire", key, ttl) + return next +end \ No newline at end of file From a3a064b486ab18faf6bc6458cca89864649138bb Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Thu, 1 Dec 2022 17:20:58 +0800 Subject: [PATCH 2/2] =?UTF-8?q?perf(=E5=AE=9A=E6=97=B6=E6=8A=A5=E5=91=8A):?= =?UTF-8?q?=20=E5=88=A0=E9=99=A4=E6=B5=8B=E8=AF=95=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/io/dataease/plugins/server/XEmailTaskServer.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java b/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java index af2b788f71..8fa6610dee 100644 --- a/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java +++ b/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java @@ -155,12 +155,6 @@ public class XEmailTaskServer { return xpackEmailCreate; } - @DeRateLimiter - @GetMapping("/testApple") - public String testApple() { - return "调用api成功"; - } - @DeRateLimiter @PostMapping(value = "/screenshot", produces = {MediaType.IMAGE_JPEG_VALUE, MediaType.IMAGE_PNG_VALUE}) public ResponseEntity screenshot(@RequestBody XpackEmailViewRequest request) {