diff --git a/rxlib-x/pom.xml b/rxlib-x/pom.xml
index e6c613a6..42ac9cd4 100644
--- a/rxlib-x/pom.xml
+++ b/rxlib-x/pom.xml
@@ -21,6 +21,10 @@
4.0.3
1.2.21
3.27.2
+ 32.1.1-jre
+
+ 1.6.2
+ 5.2.2
@@ -65,10 +69,31 @@
-
-
-
-
-
+
+ com.google.guava
+ guava
+ ${guava.version}
+
+
+
+ org.ahocorasick
+ ahocorasick
+ 0.3.1
+
+
+ org.apache.poi
+ poi
+ ${poi.version}
+
+
+ org.apache.poi
+ poi-ooxml
+ ${poi.version}
+
+
+ com.sun.mail
+ javax.mail
+ ${javaxMail.version}
+
diff --git a/rxlib-x/src/main/java/org/rx/crawler/RemoteBrowser.java b/rxlib-x/src/main/java/org/rx/crawler/RemoteBrowser.java
index d0a4ee74..34a5dd95 100644
--- a/rxlib-x/src/main/java/org/rx/crawler/RemoteBrowser.java
+++ b/rxlib-x/src/main/java/org/rx/crawler/RemoteBrowser.java
@@ -4,13 +4,13 @@
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.rx.bean.FlagsEnum;
-import org.rx.spring.MiddlewareConfig;
import org.rx.core.Linq;
import org.rx.core.Reflects;
import org.rx.exception.InvalidException;
import org.rx.net.Sockets;
import org.rx.net.rpc.Remoting;
import org.rx.net.rpc.RpcClientConfig;
+import org.rx.spring.MiddlewareConfig;
import org.rx.spring.SpringContext;
import org.rx.util.function.*;
diff --git a/rxlib-x/src/main/java/org/rx/jdbc/JdbcExecutor.java b/rxlib-x/src/main/java/org/rx/jdbc/JdbcExecutor.java
index 6b63113b..1ca021f6 100644
--- a/rxlib-x/src/main/java/org/rx/jdbc/JdbcExecutor.java
+++ b/rxlib-x/src/main/java/org/rx/jdbc/JdbcExecutor.java
@@ -25,7 +25,8 @@
import java.util.List;
import static org.rx.core.Extends.*;
-import static org.rx.core.Sys.*;
+import static org.rx.core.Sys.proxy;
+import static org.rx.core.Sys.toJsonString;
@Slf4j
public class JdbcExecutor extends Disposable implements EventPublisher, JdbcExecutable {
diff --git a/rxlib-x/src/main/java/org/rx/redis/RateLimiterAdapter.java b/rxlib-x/src/main/java/org/rx/redis/RateLimiterAdapter.java
new file mode 100644
index 00000000..9db81b1f
--- /dev/null
+++ b/rxlib-x/src/main/java/org/rx/redis/RateLimiterAdapter.java
@@ -0,0 +1,5 @@
+package org.rx.redis;
+
+public interface RateLimiterAdapter {
+ boolean tryAcquire();
+}
diff --git a/rxlib-x/src/main/java/org/rx/redis/RedisCache.java b/rxlib-x/src/main/java/org/rx/redis/RedisCache.java
index 80a13193..29fdc385 100644
--- a/rxlib-x/src/main/java/org/rx/redis/RedisCache.java
+++ b/rxlib-x/src/main/java/org/rx/redis/RedisCache.java
@@ -11,10 +11,15 @@
import org.redisson.config.Config;
import org.rx.bean.BiTuple;
import org.rx.codec.CodecUtil;
-import org.rx.core.*;
+import org.rx.core.Cache;
+import org.rx.core.CachePolicy;
+import org.rx.core.RxConfig;
+import org.rx.core.Tasks;
import org.rx.util.function.BiFunc;
-import java.util.*;
+import java.util.AbstractMap;
+import java.util.HashSet;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.rx.core.Extends.quietly;
diff --git a/rxlib-x/src/main/java/org/rx/redis/RedisRateLimiter.java b/rxlib-x/src/main/java/org/rx/redis/RedisRateLimiter.java
new file mode 100644
index 00000000..6f2ef919
--- /dev/null
+++ b/rxlib-x/src/main/java/org/rx/redis/RedisRateLimiter.java
@@ -0,0 +1,55 @@
+package org.rx.redis;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.RRateLimiter;
+import org.redisson.api.RateIntervalUnit;
+import org.redisson.api.RateLimiterConfig;
+import org.redisson.api.RateType;
+import org.rx.core.Constants;
+
+@Slf4j
+public class RedisRateLimiter implements RateLimiterAdapter {
+ final RedisCache rCache;
+ final RRateLimiter limiter;
+
+ public int getPermitsPerSecond() {
+ return limiter.getConfig().getRate().intValue();
+ }
+
+ @Override
+ public boolean tryAcquire() {
+ return limiter.tryAcquire();
+ }
+
+ public RedisRateLimiter(@NonNull RedisCache rCache, String acquireKey) {
+ this(rCache, acquireKey, Constants.CPU_THREADS);
+ }
+
+ public RedisRateLimiter(@NonNull RedisCache rCache, String acquireKey, int permitsPerSecond) {
+ this.rCache = rCache;
+ limiter = createLimiter(acquireKey, permitsPerSecond, 1);
+ }
+
+ RRateLimiter createLimiter(String key, long rate, long rateInterval) {
+ RRateLimiter limiter = rCache.getClient().getRateLimiter(key);
+ if (limiter.isExists()) {
+ RateLimiterConfig config = limiter.getConfig();
+ if (config.getRate() == rate && config.getRateInterval() == RateIntervalUnit.SECONDS.toMillis(rateInterval)) {
+ return limiter;
+ }
+ }
+
+ log.info("trySetRate start, {} {}", key, rate);
+ int retry = 4;
+ // 循环直到重新配置成功
+ while (--retry > 0 && !limiter.trySetRate(RateType.OVERALL, rate, rateInterval, RateIntervalUnit.SECONDS)) {
+ limiter.delete();
+ limiter = rCache.getClient().getRateLimiter(key);
+ }
+ if (retry == 0) {
+ log.warn("trySetRate fail, {} {}", key, rate);
+ }
+ return limiter;
+ }
+}
diff --git a/rxlib-x/src/main/java/org/rx/redis/RedisUtil.java b/rxlib-x/src/main/java/org/rx/redis/RedisUtil.java
index aafe9eac..41390c42 100644
--- a/rxlib-x/src/main/java/org/rx/redis/RedisUtil.java
+++ b/rxlib-x/src/main/java/org/rx/redis/RedisUtil.java
@@ -1,8 +1,7 @@
package org.rx.redis;
-import lombok.RequiredArgsConstructor;
+import com.google.common.util.concurrent.RateLimiter;
import org.redisson.api.RLock;
-import org.redisson.api.RedissonClient;
import org.rx.core.Cache;
import org.rx.core.Strings;
import org.rx.core.Sys;
@@ -22,7 +21,14 @@ public static Lock wrapLock(RLock rLock) {
});
}
- public static Cache wrapCache(String redisUrl) {
- return Sys.fallbackProxy(Cache.class, new RedisCache<>(redisUrl), new Lazy<>(() -> Cache.getInstance(MemoryCache.class)));
+ public static Cache wrapCache(RedisCache rCache) {
+ return Sys.fallbackProxy(Cache.class, rCache, new Lazy<>(() -> Cache.getInstance(MemoryCache.class)));
+ }
+
+ public static RateLimiterAdapter wrapRateLimiter(RedisRateLimiter rRateLimiter) {
+ return Sys.fallbackProxy(RateLimiterAdapter.class, rRateLimiter, new Lazy<>(() -> {
+ RateLimiter limiter = RateLimiter.create(rRateLimiter.getPermitsPerSecond());
+ return () -> limiter.tryAcquire();
+ }));
}
}
diff --git a/rxlib-x/src/main/java/org/rx/spring/BeanRegister.java b/rxlib-x/src/main/java/org/rx/spring/BeanRegister.java
index 01b693d3..1725bf7e 100644
--- a/rxlib-x/src/main/java/org/rx/spring/BeanRegister.java
+++ b/rxlib-x/src/main/java/org/rx/spring/BeanRegister.java
@@ -1,15 +1,19 @@
package org.rx.spring;
import lombok.extern.slf4j.Slf4j;
-import org.rx.core.Cache;
-import org.rx.core.IOC;
-import org.rx.core.Strings;
+import org.rx.core.*;
import org.rx.exception.InvalidException;
+import org.rx.redis.RateLimiterAdapter;
+import org.rx.redis.RedisCache;
+import org.rx.redis.RedisRateLimiter;
import org.rx.redis.RedisUtil;
+import org.rx.util.Servlets;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+import static org.rx.core.Extends.ifNull;
+
@Configuration
@Slf4j
public class BeanRegister {
@@ -17,14 +21,33 @@ public class BeanRegister {
@Bean
@ConditionalOnProperty(name = REDIS_PROP_NAME)
- public Cache redisCache(MiddlewareConfig redisConfig) {
- if (Strings.isEmpty(redisConfig.getRedisUrl())) {
+ public RedisCache redisCache(MiddlewareConfig conf) {
+ if (Strings.isEmpty(conf.getRedisUrl())) {
throw new InvalidException("app.redisUrl is null");
}
- Cache cache = RedisUtil.wrapCache(redisConfig.getRedisUrl());
- IOC.register(Cache.class, cache);
+ RedisCache cache = new RedisCache<>(conf.getRedisUrl());
+ IOC.register(Cache.class, RedisUtil.wrapCache(cache));
log.info("register RedisCache ok");
return cache;
}
+
+ @Bean
+ @ConditionalOnProperty(name = REDIS_PROP_NAME)
+ public RateLimiterAdapter httpRateLimiterAdapter(RedisCache rCache, MiddlewareConfig conf) {
+ String[] acquireWhiteList = conf.getLimiterWhiteList();
+ return () -> {
+ String clientIp = ifNull(Servlets.requestIp(), "ALL");
+
+ if (!Arrays.isEmpty(acquireWhiteList)
+ && Linq.from(acquireWhiteList).any(p -> Strings.startsWith(clientIp, p))) {
+ return true;
+ }
+
+ String rk = "RateLimiter:" + clientIp;
+ RateLimiterAdapter adapter = IOC.weakMap(false)
+ .computeIfAbsent(rk, k -> RedisUtil.wrapRateLimiter(new RedisRateLimiter(rCache, k, conf.getLimiterPermits())));
+ return adapter.tryAcquire();
+ };
+ }
}
diff --git a/rxlib-x/src/main/java/org/rx/spring/MiddlewareConfig.java b/rxlib-x/src/main/java/org/rx/spring/MiddlewareConfig.java
index f1e2ddaa..18a75dfa 100644
--- a/rxlib-x/src/main/java/org/rx/spring/MiddlewareConfig.java
+++ b/rxlib-x/src/main/java/org/rx/spring/MiddlewareConfig.java
@@ -2,8 +2,6 @@
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.core.Ordered;
-import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
@Data
@@ -13,6 +11,7 @@
public class MiddlewareConfig {
private String redisUrl;
private String storeUrl;
+
private int limiterPermits = 12;
private String limiterWhiteList;
diff --git a/rxlib-x/src/main/java/org/rx/util/Helper.java b/rxlib-x/src/main/java/org/rx/util/Helper.java
new file mode 100644
index 00000000..df958d4e
--- /dev/null
+++ b/rxlib-x/src/main/java/org/rx/util/Helper.java
@@ -0,0 +1,194 @@
+package org.rx.util;
+
+import lombok.NonNull;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.apache.poi.ss.usermodel.*;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.rx.core.Numbers;
+import org.rx.spring.MiddlewareConfig;
+import org.rx.spring.SpringContext;
+
+import javax.mail.*;
+import javax.mail.internet.InternetAddress;
+import javax.mail.internet.MimeMessage;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.function.Function;
+
+import static org.rx.core.Extends.eq;
+import static org.rx.core.Sys.toJsonString;
+
+@Slf4j
+public class Helper {
+ public static void sendEmail(String body) {
+ MiddlewareConfig config = SpringContext.getBean(MiddlewareConfig.class);
+ Helper.sendEmail(body, config.getSmtpPwd(), config.getSmtpTo());
+ }
+
+ public static void sendEmail(String body, @NonNull String password, @NonNull String toEmail) {
+ final String fromEmail = "17091916400@163.com";
+ Properties props = new Properties();
+ props.put("mail.smtp.host", "smtp.163.com");
+ props.put("mail.smtp.port", "25");
+ props.put("mail.smtp.auth", "true");
+ Session session = Session.getInstance(props, new Authenticator() {
+ protected PasswordAuthentication getPasswordAuthentication() {
+ return new PasswordAuthentication(fromEmail, password);
+ }
+ });
+
+ try {
+ MimeMessage msg = new MimeMessage(session);
+ msg.addHeader("Content-type", "text/HTML; charset=UTF-8");
+ msg.addHeader("format", "flowed");
+ msg.addHeader("Content-Transfer-Encoding", "8bit");
+
+ msg.setFrom(new InternetAddress(fromEmail, "System"));
+ msg.setReplyTo(InternetAddress.parse("no_reply@f-li.cn", false));
+
+ msg.setSubject("Notification", "UTF-8");
+ msg.setText(body, "UTF-8");
+ msg.setSentDate(new Date());
+
+ msg.setRecipients(Message.RecipientType.TO, InternetAddress.parse(toEmail, false));
+ Transport.send(msg);
+ } catch (Exception e) {
+ log.warn("sendEmail {}", e.getMessage());
+ }
+ }
+
+ public static Map> readExcel(InputStream in, boolean is2003File) {
+ return readExcel(in, is2003File, false, false);
+ }
+
+ @SneakyThrows
+ public static Map> readExcel(InputStream in, boolean is2003File, boolean skipColumn, boolean keepNullRow) {
+ Map> data = new LinkedHashMap<>();
+ FormulaEvaluator evaluator = null;
+ try (Workbook workbook = is2003File ? new HSSFWorkbook(in) : new XSSFWorkbook(in)) {
+ for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) {
+ List rows = new ArrayList<>();
+ Sheet sheet = workbook.getSheetAt(sheetIndex);
+ for (int rowIndex = skipColumn ? 1 : sheet.getFirstRowNum(); rowIndex <= sheet.getLastRowNum(); rowIndex++) {
+ Row row = sheet.getRow(rowIndex);
+ if (row == null) {
+ if (keepNullRow) {
+ rows.add(null);
+ }
+ continue;
+ }
+ List