优化商品索引的促销信息生成。优化批量生成分词

config/application.yml

@@ -29,7 +29,7 @@ spring:
     type: redis
   # Redis
   redis:
-    host: 192.168.0.116
+    host: 127.0.0.1
     port: 6379
     password: lilishop
     lettuce:
@@ -60,7 +60,7 @@ spring:
       default-datasource:
         type: com.alibaba.druid.pool.DruidDataSource
         driverClassName: com.mysql.cj.jdbc.Driver
-        url: jdbc:mysql://192.168.0.116:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
+        url: jdbc:mysql://127.0.0.1:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
         username: root
         password: lilishop
         maxActive: 50
@@ -119,6 +119,7 @@ ignored:
     - /store/passport/login/refresh/**
     - /common/common/slider/**
     - /common/common/sms/**
+    - /common/common/site
     - /buyer/payment/cashier/**
     - /buyer/other/pageData/**
     - /buyer/other/article/**
@@ -268,7 +269,7 @@ lili:
   data:
     elasticsearch:
       cluster-name: elasticsearch
-      cluster-nodes: 192.168.0.116:9200
+      cluster-nodes: 127.0.0.1:9200
       index:
         number-of-replicas: 0
         number-of-shards: 3
@@ -279,7 +280,7 @@ lili:
     #        password: LiLiShopES
 
     logstash:
-      server: 192.168.0.116:4560
+      server: 127.0.0.1:4560
     rocketmq:
       promotion-topic: lili_promotion_topic
       promotion-group: lili_promotion_group
@@ -300,7 +301,7 @@ lili:
       after-sale-topic: lili_after_sale_topic
       after-sale-group: lili_after_sale_group
 rocketmq:
-  name-server: 192.168.0.116:9876
+  name-server: 127.0.0.1:9876
   isVIPChannel: false
   producer:
     group: lili_group
@@ -309,7 +310,7 @@ rocketmq:
 xxl:
   job:
     admin:
-      addresses: http://192.168.0.116:9001/xxl-job-admin
+      addresses: http://127.0.0.1:9001/xxl-job-admin
     executor:
       appname: xxl-job-executor-lilishop
       address:

consumer/src/main/java/cn/lili/listener/GoodsMessageListener.java

@@ -275,8 +275,6 @@ private void updateGoodsIndexPromotions(String promotionsJsonStr) {
                 this.goodsIndexService.deleteEsGoodsPromotionByPromotionKey(skuIds, esPromotionKey);
                 this.goodsIndexService.updateEsGoodsIndexPromotions(skuIds, promotions, esPromotionKey);
             } else if (PromotionsScopeTypeEnum.ALL.name().equals(promotions.getScopeType())) {
-                // 更新商品索引促销信息（删除原索引中相关的促销信息，更新索引中促销信息）
-                this.goodsIndexService.deleteEsGoodsPromotionByPromotionKey(esPromotionKey);
                 this.goodsIndexService.updateEsGoodsIndexAllByList(promotions, esPromotionKey);
             }
         } catch (Exception e) {

consumer/src/main/resources/application.yml

@@ -67,7 +67,7 @@ spring:
       default-datasource:
         type: com.alibaba.druid.pool.DruidDataSource
         driverClassName: com.mysql.cj.jdbc.Driver
-        url: jdbc:mysql://127.0.0.1:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
+        url: jdbc:mysql://127.0.0.1:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
         username: root
         password: lilishop
         maxActive: 20

framework/src/main/java/cn/lili/modules/search/entity/dos/CustomWords.java

@@ -4,8 +4,10 @@
 import com.baomidou.mybatisplus.annotation.TableName;
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
 import org.hibernate.validator.constraints.Length;
 
 import javax.validation.constraints.NotEmpty;
@@ -20,6 +22,8 @@
 @TableName("li_custom_words")
 @ApiModel(value = "自定义分词")
 @EqualsAndHashCode(callSuper = true)
+@NoArgsConstructor
+@AllArgsConstructor
 public class CustomWords extends BaseEntity {
 
     private static final long serialVersionUID = 650889506808657977L;

framework/src/main/java/cn/lili/modules/search/mapper/CustomWordsMapper.java

@@ -1,6 +1,7 @@
 package cn.lili.modules.search.mapper;
 
 import cn.lili.modules.search.entity.dos.CustomWords;
+import cn.lili.mybatis.mybatisplus.SpiceBaseMapper;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 
 /**
@@ -9,5 +10,5 @@
  * @author paulG
  * @since 2020/10/15
  **/
-public interface CustomWordsMapper extends BaseMapper<CustomWords> {
+public interface CustomWordsMapper extends BaseMapper<CustomWords>, SpiceBaseMapper<CustomWords> {
 }

framework/src/main/java/cn/lili/modules/search/service/CustomWordsService.java

@@ -6,6 +6,8 @@
 import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.service.IService;
 
+import java.util.List;
+
 /**
  * 自定义分词业务层
  *
@@ -52,6 +54,16 @@ public interface CustomWordsService extends IService<CustomWords> {
      */
     boolean deleteCustomWords(String id);
 
+    /**
+     * 根据名字批量删除
+     *
+     * @param names 名称列表
+     * @return 是否删除成功
+     */
+    boolean deleteBathByName(List<String> names);
+
+    long insertBatchCustomWords(List<CustomWords> customWordsList);
+
     /**
      * 分页查询自定义分词
      *

framework/src/main/java/cn/lili/modules/search/serviceimpl/CustomWordsServiceImpl.java

@@ -91,6 +91,18 @@ public boolean deleteCustomWords(String id) {
         return this.removeById(id);
     }
 
+    @Override
+    public boolean deleteBathByName(List<String> names) {
+        LambdaQueryWrapper<CustomWords> queryWrapper = new LambdaQueryWrapper<>();
+        queryWrapper.in(CustomWords::getName, names);
+        return this.remove(queryWrapper);
+    }
+
+    @Override
+    public long insertBatchCustomWords(List<CustomWords> customWordsList) {
+        return this.baseMapper.insertBatchSomeColumn(customWordsList);
+    }
+
     /**
      * 修改自定义分词
      *

framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsIndexServiceImpl.java

@@ -15,6 +15,7 @@
 import cn.lili.common.exception.RetryException;
 import cn.lili.common.exception.ServiceException;
 import cn.lili.common.properties.RocketmqCustomProperties;
+import cn.lili.common.vo.PageVO;
 import cn.lili.elasticsearch.BaseElasticsearchService;
 import cn.lili.elasticsearch.EsSuffix;
 import cn.lili.elasticsearch.config.ElasticsearchProperties;
@@ -35,10 +36,10 @@
 import cn.lili.modules.promotion.entity.enums.PromotionsStatusEnum;
 import cn.lili.modules.promotion.service.PromotionService;
 import cn.lili.modules.promotion.tools.PromotionTools;
+import cn.lili.modules.search.entity.dos.CustomWords;
 import cn.lili.modules.search.entity.dos.EsGoodsAttribute;
 import cn.lili.modules.search.entity.dos.EsGoodsIndex;
 import cn.lili.modules.search.entity.dto.EsGoodsSearchDTO;
-import cn.lili.modules.search.entity.vo.CustomWordsVO;
 import cn.lili.modules.search.repository.EsGoodsIndexRepository;
 import cn.lili.modules.search.service.CustomWordsService;
 import cn.lili.modules.search.service.EsGoodsIndexService;
@@ -50,7 +51,6 @@
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.rocketmq.spring.core.RocketMQTemplate;
-import org.assertj.core.util.IterableUtil;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.update.UpdateRequest;
@@ -65,8 +65,8 @@
 import org.elasticsearch.index.reindex.UpdateByQueryRequest;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
-import org.mybatis.spring.MyBatisSystemException;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.PageRequest;
 import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
 import org.springframework.data.elasticsearch.core.SearchHit;
 import org.springframework.data.elasticsearch.core.SearchPage;
@@ -287,17 +287,28 @@ private void analyzeAndSaveWords(EsGoodsIndex goods) {
             AnalyzeRequest analyzeRequest = AnalyzeRequest.withIndexAnalyzer(getIndexName(), "ik_max_word", goods.getGoodsName());
             AnalyzeResponse analyze = client.indices().analyze(analyzeRequest, RequestOptions.DEFAULT);
             List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
-
+            List<CustomWords> customWordsList = new ArrayList<>();
+            List<String> keywordsList = new ArrayList<>();
             if (goods.getAttrList() != null && !goods.getAttrList().isEmpty()) {
                 //保存分词
                 for (EsGoodsAttribute esGoodsAttribute : goods.getAttrList()) {
-                    wordsToDb(esGoodsAttribute.getValue());
+                    if (keywordsList.stream().noneMatch(i -> i.toLowerCase(Locale.ROOT).equals(esGoodsAttribute.getValue().toLowerCase(Locale.ROOT)))) {
+                        keywordsList.add(esGoodsAttribute.getValue());
+                        customWordsList.add(new CustomWords(esGoodsAttribute.getValue(), 0));
+                    }
                 }
             }
             //分析词条
             for (AnalyzeResponse.AnalyzeToken token : tokens) {
+                if (keywordsList.stream().noneMatch(i -> i.toLowerCase(Locale.ROOT).equals(token.getTerm().toLowerCase(Locale.ROOT)))) {
+                    keywordsList.add(token.getTerm());
+                    customWordsList.add(new CustomWords(token.getTerm(), 0));
+                }
                 //保存词条进入数据库
-                wordsToDb(token.getTerm());
+            }
+            if (CollUtil.isNotEmpty(customWordsList)) {
+                customWordsService.deleteBathByName(keywordsList);
+                customWordsService.insertBatchCustomWords(customWordsList);
             }
         } catch (IOException e) {
             log.info(goods + "分词错误", e);
@@ -537,25 +548,35 @@ public void updateEsGoodsIndexByList(List<PromotionGoods> promotionGoodsList, Ba
     @Override
     public void updateEsGoodsIndexAllByList(BasePromotions promotion, String key) {
         ThreadUtil.execAsync(() -> {
-            List<EsGoodsIndex> goodsIndices = new ArrayList<>();
-            //如果storeId不为空，则表示是店铺活动
-            if (promotion.getStoreId() != null && !promotion.getStoreId().equals(PromotionTools.PLATFORM_ID)) {
-                EsGoodsSearchDTO searchDTO = new EsGoodsSearchDTO();
-                searchDTO.setStoreId(promotion.getStoreId());
-                //查询出店铺商品
-                SearchPage<EsGoodsIndex> esGoodsIndices = goodsSearchService.searchGoods(searchDTO, null);
-                for (SearchHit<EsGoodsIndex> searchHit : esGoodsIndices.getContent()) {
-                    goodsIndices.add(searchHit.getContent());
+            for (int i = 1; ; i++) {
+                List<String> skuIds;
+                //如果storeId不为空，则表示是店铺活动
+                if (promotion.getStoreId() != null && !promotion.getStoreId().equals(PromotionTools.PLATFORM_ID)) {
+                    PageVO pageVO = new PageVO();
+                    pageVO.setPageNumber(i);
+                    pageVO.setPageSize(1000);
+                    EsGoodsSearchDTO searchDTO = new EsGoodsSearchDTO();
+                    searchDTO.setStoreId(promotion.getStoreId());
+                    //查询出店铺商品
+                    SearchPage<EsGoodsIndex> esGoodsIndices = goodsSearchService.searchGoods(searchDTO, pageVO);
+
+                    skuIds = esGoodsIndices.isEmpty() ? new ArrayList<>() : esGoodsIndices.getContent().stream().map(SearchHit::getId).collect(Collectors.toList());
+                } else {
+                    //否则是平台活动
+                    org.springframework.data.domain.Page<EsGoodsIndex> all = goodsIndexRepository.findAll(PageRequest.of(i, 1000));
+
+                    //查询出全部商品
+                    skuIds = all.isEmpty() ? new ArrayList<>() : all.toList().stream().map(EsGoodsIndex::getId).collect(Collectors.toList());
                 }
-            } else {
-                //否则是平台活动
-                Iterable<EsGoodsIndex> all = goodsIndexRepository.findAll();
-                //查询出全部商品
-                goodsIndices = new ArrayList<>(IterableUtil.toCollection(all));
+                if (skuIds.isEmpty()) {
+                    break;
+                }
+                this.deleteEsGoodsPromotionByPromotionKey(skuIds, key);
+                this.updateEsGoodsIndexPromotions(skuIds, promotion, key);
             }
-            List<String> skuIds = goodsIndices.stream().map(EsGoodsIndex::getId).collect(Collectors.toList());
-            this.updateEsGoodsIndexPromotions(skuIds, promotion, key);
+
         });
+
     }
 
     @Override
@@ -730,17 +751,16 @@ private UpdateRequest updateGoodsIndexPromotion(EsGoodsIndex goodsIndex, String
         } else {
             promotionMap = goodsIndex.getOriginPromotionMap();
         }
-
-        log.info("ES修改商品活动索引-原商品索引信息:{}", goodsIndex);
-        log.info("ES修改商品活动索引-原商品索引活动信息:{}", promotionMap);
+//        log.info("ES修改商品活动索引-原商品索引信息:{}", goodsIndex);
+//        log.info("ES修改商品活动索引-原商品索引活动信息:{}", promotionMap);
         //如果活动已结束
         if (promotion.getPromotionStatus().equals(PromotionsStatusEnum.END.name()) || promotion.getPromotionStatus().equals(PromotionsStatusEnum.CLOSE.name())) {//如果存在活动
             //删除活动
             promotionMap.remove(key);
         } else {
             promotionMap.put(key, promotion);
         }
-        log.info("ES修改商品活动索引-过滤后商品索引活动信息:{}", promotionMap);
+//        log.info("ES修改商品活动索引-过滤后商品索引活动信息:{}", promotionMap);
         return this.getGoodsIndexPromotionUpdateRequest(goodsIndex.getId(), promotionMap);
     }
 
@@ -759,7 +779,6 @@ private UpdateRequest getGoodsIndexPromotionUpdateRequest(String id, Map<String,
         Map<String, Object> params = new HashMap<>();
         params.put("promotionMap", JSONUtil.toJsonStr(promotionMap));
         Script script = new Script(ScriptType.INLINE, "painless", "ctx._source.promotionMapJson=params.promotionMap;", params);
-        log.info("执行脚本内容：{}", script);
         updateRequest.script(script);
         return updateRequest;
     }
@@ -812,25 +831,6 @@ private void removePromotionKey(String currentKey, Map<String, Object> promotion
         }
     }
 
-    /**
-     * 将商品关键字入库
-     *
-     * @param words 商品关键字
-     */
-    private void wordsToDb(String words) {
-        if (CharSequenceUtil.isEmpty(words)) {
-            return;
-        }
-        try {
-            //是否有重复
-            customWordsService.addCustomWords(new CustomWordsVO(words));
-        } catch (MyBatisSystemException me) {
-            log.error(words + "关键字已存在！", me);
-        } catch (Exception e) {
-            log.error("关键字入库异常！", e);
-        }
-    }
-
     private String getIndexName() {
         //索引名称拼接
         return elasticsearchProperties.getIndexPrefix() + "_" + EsSuffix.GOODS_INDEX_NAME;

framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceBaseMapper.java

@@ -0,0 +1,19 @@
+package cn.lili.mybatis.mybatisplus;
+
+import java.util.List;
+
+/**
+ * @author paulG
+ * @since 2022/7/18
+ **/
+public interface SpiceBaseMapper<T> {
+
+    /**
+     * 批量插入
+     * {@link com.baomidou.mybatisplus.extension.injector.methods.InsertBatchSomeColumn}
+     *
+     * @param entityList 要插入的数据
+     * @return 成功插入的数据条数
+     */
+    long insertBatchSomeColumn(List<T> entityList);
+}

framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceSqlInjector.java

@@ -0,0 +1,33 @@
+package cn.lili.mybatis.mybatisplus;
+
+import com.baomidou.mybatisplus.core.injector.AbstractMethod;
+import com.baomidou.mybatisplus.core.injector.DefaultSqlInjector;
+import com.baomidou.mybatisplus.core.metadata.TableInfo;
+import com.baomidou.mybatisplus.extension.injector.methods.InsertBatchSomeColumn;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+/**
+ * @author paulG
+ * @since 2022/7/18
+ **/
+@Component
+public class SpiceSqlInjector extends DefaultSqlInjector {
+
+    /**
+     * 如果只需增加方法，保留mybatis plus自带方法，
+     * 可以先获取super.getMethodList()，再添加add
+     */
+    @Override
+    public List<AbstractMethod> getMethodList(Class<?> mapperClass, TableInfo tableInfo) {
+        // 注意：此SQL注入器继承了DefaultSqlInjector(默认注入器)，调用了DefaultSqlInjector的getMethodList方法，保留了mybatis-plus的自带方法
+        List<AbstractMethod> methodList = super.getMethodList(mapperClass, tableInfo);
+        // 注入InsertBatchSomeColumn
+        // 在!t.isLogicDelete()表示不要逻辑删除字段，!"update_time".equals(t.getColumn())表示不要字段名为 update_time 的字段,不对进行操作
+        // methodList.add(new InsertBatchSomeColumn(t -> !t.isLogicDelete() && !"update_time".equals(t.getColumn())));
+        // 要逻辑删除 t.isLogicDelete() 默认不要
+        methodList.add(new InsertBatchSomeColumn(t -> !t.isLogicDelete()));
+        return methodList;
+    }
+}