yudao
/
spring-cloud
mirror of https://gitee.com/zhijiantianya/yudao-cloud.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

敏感词的本地缓存，使用 Job 轮询，替代 MQ 广播

pull/46/head
YunaiV 2023-07-29 07:32:06 +08:00
parent 0b17298963
commit e205129943
7 changed files with 48 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/dal/mysql/sensitiveword/SensitiveWordMapper.java
View File

@ -7,7 +7,9 @@ import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.Sensitiv
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordPageReqVO;
import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Select;
import java.time.LocalDateTime;
import java.util.List;
/**
@ -40,4 +42,7 @@ public interface SensitiveWordMapper extends BaseMapperX<SensitiveWordDO> {
return selectOne(SensitiveWordDO::getName, name);
}
@Select("SELECT COUNT(*) FROM system_sensitive_word WHERE update_time > #{maxUpdateTime}")
Long selectCountByUpdateTimeGt(LocalDateTime maxTime);
}

29
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/mq/consumer/sensitiveword/SensitiveWordRefreshConsumer.java
View File

@ -1,29 +0,0 @@
package cn.iocoder.yudao.module.system.mq.consumer.sensitiveword;
import cn.iocoder.yudao.module.system.mq.message.sensitiveword.SensitiveWordRefreshMessage;
import cn.iocoder.yudao.module.system.service.sensitiveword.SensitiveWordService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
/**
* 针对 {@link SensitiveWordRefreshMessage} 的消费者
*
* @author 芋道源码
*/
@Component
@Slf4j
public class SensitiveWordRefreshConsumer {
@Resource
private SensitiveWordService sensitiveWordService;
@EventListener
public void execute(SensitiveWordRefreshMessage message) {
log.info("[execute][收到 SensitiveWord 刷新消息]");
sensitiveWordService.initLocalCache();
}
}

21
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/mq/message/sensitiveword/SensitiveWordRefreshMessage.java
View File

@ -1,21 +0,0 @@
package cn.iocoder.yudao.module.system.mq.message.sensitiveword;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.springframework.cloud.bus.event.RemoteApplicationEvent;
/**
* 敏感词的刷新 Message
*/
@Data
@EqualsAndHashCode(callSuper = true)
public class SensitiveWordRefreshMessage extends RemoteApplicationEvent {
public SensitiveWordRefreshMessage() {
}
public SensitiveWordRefreshMessage(Object source, String originService, String destinationService) {
super(source, originService, DEFAULT_DESTINATION_FACTORY.getDestination(destinationService));
}
}

21
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/mq/producer/sensitiveword/SensitiveWordProducer.java
View File

@ -1,21 +0,0 @@
package cn.iocoder.yudao.module.system.mq.producer.sensitiveword;
import cn.iocoder.yudao.framework.mq.core.bus.AbstractBusProducer;
import cn.iocoder.yudao.module.system.mq.message.sensitiveword.SensitiveWordRefreshMessage;
import org.springframework.stereotype.Component;
/**
* 敏感词相关的 Producer
*/
@Component
public class SensitiveWordProducer extends AbstractBusProducer {{
}
/**
* 发送 {@link SensitiveWordRefreshMessage} 消息
*/
public void sendSensitiveWordRefreshMessage() {
publishEvent(new SensitiveWordRefreshMessage(this, getBusId(), selfDestinationService()));
}
}

5
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/service/sensitiveword/SensitiveWordService.java
View File

@ -18,11 +18,6 @@ import java.util.Set;
*/
public interface SensitiveWordService {
/**
* 初始化本地缓存
*/
void initLocalCache();
/**
* 创建敏感词
*

50
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/service/sensitiveword/SensitiveWordServiceImpl.java
View File

@ -11,21 +11,24 @@ import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.Sensitiv
import cn.iocoder.yudao.module.system.convert.sensitiveword.SensitiveWordConvert;
import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
import cn.iocoder.yudao.module.system.dal.mysql.sensitiveword.SensitiveWordMapper;
import cn.iocoder.yudao.module.system.mq.producer.sensitiveword.SensitiveWordProducer;
import cn.iocoder.yudao.module.system.util.collection.SimpleTrie;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.validation.annotation.Validated;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.TimeUnit;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.filterList;
import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.getMaxValue;
import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_WORD_EXISTS;
import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_WORD_NOT_EXISTS;
@ -39,6 +42,11 @@ import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_
@Validated
public class SensitiveWordServiceImpl implements SensitiveWordService {
/**
* 敏感词列表缓存
*/
@Getter
private volatile List<SensitiveWordDO> sensitiveWordCache = Collections.emptyList();
/**
* 敏感词标签缓存
* key：敏感词编号 {@link SensitiveWordDO#getId()}
@ -51,9 +59,6 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
@Resource
private SensitiveWordMapper sensitiveWordMapper;
@Resource
private SensitiveWordProducer sensitiveWordProducer;
/**
* 默认的敏感词的字典树，包含所有敏感词
*/
@ -68,7 +73,6 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
/**
* 初始化缓存
*/
@Override
@PostConstruct
public void initLocalCache() {
// 第一步：查询数据
@ -80,6 +84,7 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
Set<String> tags = new HashSet<>();
sensitiveWords.forEach(word -> tags.addAll(word.getTags()));
sensitiveWordTagsCache = tags;
sensitiveWordCache = sensitiveWords;
// 写入 defaultSensitiveWordTrie、tagSensitiveWordTries 缓存
initSensitiveWordTrie(sensitiveWords);
}
@ -105,6 +110,26 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
this.tagSensitiveWordTries = tagSensitiveWordTries;
}
/**
* 通过定时任务轮询，刷新缓存
*
* 目的：多节点部署时，通过轮询”通知“所有节点，进行刷新
*/
@Scheduled(initialDelay = 60, fixedRate = 60, timeUnit = TimeUnit.SECONDS)
public void refreshLocalCache() {
// 情况一：如果缓存里没有数据，则直接刷新缓存
if (CollUtil.isEmpty(sensitiveWordCache)) {
initLocalCache();
return;
}
// 情况二，如果缓存里数据，则通过 updateTime 判断是否有数据变更，有变更则刷新缓存
LocalDateTime maxTime = getMaxValue(sensitiveWordCache, SensitiveWordDO::getUpdateTime);
if (sensitiveWordMapper.selectCountByUpdateTimeGt(maxTime) > 0) {
initLocalCache();
}
}
@Override
public Long createSensitiveWord(SensitiveWordCreateReqVO createReqVO) {
// 校验唯一性
@ -113,8 +138,9 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
// 插入
SensitiveWordDO sensitiveWord = SensitiveWordConvert.INSTANCE.convert(createReqVO);
sensitiveWordMapper.insert(sensitiveWord);
// 发送消息，刷新缓存
sensitiveWordProducer.sendSensitiveWordRefreshMessage();
// 刷新缓存
initLocalCache();
return sensitiveWord.getId();
}
@ -127,8 +153,9 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
// 更新
SensitiveWordDO updateObj = SensitiveWordConvert.INSTANCE.convert(updateReqVO);
sensitiveWordMapper.updateById(updateObj);
// 发送消息，刷新缓存
sensitiveWordProducer.sendSensitiveWordRefreshMessage();
// 刷新缓存
initLocalCache();
}
@Override
@ -137,8 +164,9 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
validateSensitiveWordExists(id);
// 删除
sensitiveWordMapper.deleteById(id);
// 发送消息，刷新缓存
sensitiveWordProducer.sendSensitiveWordRefreshMessage();
// 刷新缓存
initLocalCache();
}
private void validateSensitiveWordNameUnique(Long id, String name) {

13
yudao-module-system/yudao-module-system-biz/src/test/java/cn/iocoder/yudao/module/system/service/sensitiveword/SensitiveWordServiceImplTest.java
View File

@ -10,9 +10,7 @@ import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.Sensitiv
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordUpdateReqVO;
import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
import cn.iocoder.yudao.module.system.dal.mysql.sensitiveword.SensitiveWordMapper;
import cn.iocoder.yudao.module.system.mq.producer.sensitiveword.SensitiveWordProducer;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Import;
import javax.annotation.Resource;
@ -29,7 +27,6 @@ import static cn.iocoder.yudao.framework.test.core.util.RandomUtils.randomPojo;
import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_WORD_NOT_EXISTS;
import static java.util.Collections.singletonList;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.verify;
/**
* {@link SensitiveWordServiceImpl} 的单元测试类
@ -45,9 +42,6 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
@Resource
private SensitiveWordMapper sensitiveWordMapper;
@MockBean
private SensitiveWordProducer sensitiveWordProducer;
@Test
public void testInitLocalCache() {
SensitiveWordDO wordDO1 = randomPojo(SensitiveWordDO.class, o -> o.setName("傻瓜")
@ -61,6 +55,10 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
sensitiveWordService.initLocalCache();
// 断言 sensitiveWordTagsCache 缓存
assertEquals(SetUtils.asSet("论坛", "蔬菜"), sensitiveWordService.getSensitiveWordTagSet());
// 断言 sensitiveWordCache
assertEquals(2, sensitiveWordService.getSensitiveWordCache().size());
assertPojoEquals(wordDO1, sensitiveWordService.getSensitiveWordCache().get(0));
assertPojoEquals(wordDO2, sensitiveWordService.getSensitiveWordCache().get(1));
// 断言 tagSensitiveWordTries 缓存
assertNotNull(sensitiveWordService.getDefaultSensitiveWordTrie());
assertEquals(2, sensitiveWordService.getTagSensitiveWordTries().size());
@ -80,7 +78,6 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
// 校验记录的属性是否正确
SensitiveWordDO sensitiveWord = sensitiveWordMapper.selectById(sensitiveWordId);
assertPojoEquals(reqVO, sensitiveWord);
verify(sensitiveWordProducer).sendSensitiveWordRefreshMessage();
}
@Test
@ -98,7 +95,6 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
// 校验是否更新正确
SensitiveWordDO sensitiveWord = sensitiveWordMapper.selectById(reqVO.getId()); // 获取最新的
assertPojoEquals(reqVO, sensitiveWord);
verify(sensitiveWordProducer).sendSensitiveWordRefreshMessage();
}
@Test
@ -122,7 +118,6 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
sensitiveWordService.deleteSensitiveWord(id);
// 校验数据不存在了
assertNull(sensitiveWordMapper.selectById(id));
verify(sensitiveWordProducer).sendSensitiveWordRefreshMessage();
}
@Test