抓取数据并保存到数据库功能完成

yxt-supervise-dbcenter/docs/databases/table_create.sql

@@ -127,6 +127,7 @@ CREATE TABLE `crawl_sales_dishes`  (
 `id` int(0) NOT NULL AUTO_INCREMENT COMMENT 'id',
 
 `dataDate` varchar(100) NULL DEFAULT NULL COMMENT '数据日期',
+`storeName` varchar(100) NULL DEFAULT NULL COMMENT '门店名称',
 `dishesName` varchar(100) NULL DEFAULT NULL COMMENT '菜品名称',
 `unit` varchar(100) NULL DEFAULT NULL COMMENT '规格单位',
 `salesCount` double(12,2) NULL DEFAULT NULL COMMENT '数量',

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/pom.xml

@@ -41,16 +41,16 @@
             <groupId>com.baomidou</groupId>
             <artifactId>mybatis-plus-annotation</artifactId>
         </dependency>
+
         <dependency>
-            <groupId>junit</groupId>
+            <groupId>org.springframework.boot</groupId>
-            <artifactId>junit</artifactId>
+            <artifactId>spring-boot-starter-test</artifactId>
-            <scope>compile</scope>
+            <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>javax.servlet</groupId>
+            <groupId>org.junit.platform</groupId>
-            <artifactId>javax.servlet-api</artifactId>
+            <artifactId>junit-platform-launcher</artifactId>
-            <version>4.0.1</version>
+            <scope>test</scope>
-            <scope>compile</scope>
         </dependency>
 
 

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/YxtSuperviseDbcenterApplication.java

@@ -3,6 +3,13 @@ package com.yxt.supervise.dbcenter;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.Bean;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.concurrent.ConcurrentTaskScheduler;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 
 /**
  * @author dimengzhe
@@ -11,8 +18,20 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
         "com.yxt.common.base.config",
         "com.yxt.supervise.dbcenter"
 })
+@EnableScheduling
 public class YxtSuperviseDbcenterApplication {
     public static void main(String[] args) {
         SpringApplication.run(YxtSuperviseDbcenterApplication.class, args);
     }
+
+    @Bean
+    public ScheduledExecutorService scheduledExecutorService() {
+        ScheduledExecutorService ss = Executors.newScheduledThreadPool(10);
+        return ss;
+    }
+
+    @Bean
+    public TaskScheduler taskScheduler() {
+        return new ConcurrentTaskScheduler();
+    }
 }

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/CrawlTask.java

@@ -1,4 +1,38 @@
 package com.yxt.supervise.dbcenter.zhj.crawl;
 
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.thread.ThreadUtil;
+import com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlSalesAmount.CrawlSalesAmount;
+import com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlSalesAmount.CrawlSalesAmountService;
+import com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlSalesDishes.CrawlSalesDishesService;
+import com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlpurchase.CrawlPurchaseService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+
+@Component
 public class CrawlTask {
+
+    @Autowired
+    private CrawlSalesAmountService crawlSalesAmountService;
+    @Autowired
+    private CrawlSalesDishesService crawlSalesDishesService;
+    @Autowired
+    private CrawlPurchaseService crawlPurchaseService;
+
+
+    /**
+     * 描述 : 每天上午10点30分抓取数据并保存到数据库. <br/>
+     * <p>
+     */
+    @Scheduled(cron = "0 30 10 * * ?")
+    public void pullZhjData() {
+        String currDay = DateUtil.format(new Date(), "yyyy-MM-dd");
+        ThreadUtil.execute(() -> crawlSalesAmountService.pullData(currDay));
+        ThreadUtil.execute(() -> crawlSalesDishesService.pullData(currDay));
+        ThreadUtil.execute(() -> crawlPurchaseService.pullData(currDay));
+    }
+
 }

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/ZhjSpider.java

@@ -64,9 +64,9 @@ public class ZhjSpider {
     }
 
     public static void main(String[] args) {
-//        List<RespSalesAmount> s = ZhjSpider.crawlSalesAmount("20230508");
+       List<RespSalesAmount> s = ZhjSpider.crawlSalesAmount("20230508");
 //        List<RespSalesDishes> s = ZhjSpider.crawlSalesDishes("20230509", "熟溢香振头店");
-        List<RespPurchase> s = ZhjSpider.crawlPurchase("20230508");
+//         List<RespPurchase> s = ZhjSpider.crawlPurchase("20230508");
         System.out.println(s);
     }
 

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/biz/crawlSalesAmount/CrawlSalesAmountMapper.java

@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.Wrapper;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.core.toolkit.Constants;
+import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
 
@@ -17,4 +18,7 @@ public interface CrawlSalesAmountMapper extends BaseMapper<CrawlSalesAmount> {
     CrawlSalesAmount select();
 
     IPage<CrawlSalesAmountVo> selectPageVo(IPage<CrawlSalesAmount> page, @Param(Constants.WRAPPER) Wrapper<CrawlSalesAmount> qw);
+
+    @Delete("delete from crawl_sales_amount where dataDate=#{dataDate} ")
+    void deleteOfDay(@Param("dataDate") String currDay);
 }

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/biz/crawlSalesAmount/CrawlSalesAmountService.java

@@ -4,14 +4,18 @@ import cn.hutool.core.bean.BeanUtil;
 import com.baomidou.mybatisplus.core.conditions.Wrapper;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.yxt.common.base.service.MybatisBaseService;
 import com.yxt.common.base.utils.PagerUtil;
 import com.yxt.common.core.query.PagerQuery;
 import com.yxt.common.core.result.ResultBean;
 import com.yxt.common.core.vo.PagerVo;
+import com.yxt.supervise.dbcenter.zhj.crawl.ZhjSpider;
+import com.yxt.supervise.dbcenter.zhj.crawl.spider.RespSalesAmount;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -19,16 +23,13 @@ import java.util.List;
  * @create 2023-05-09-10:07
  */
 @Service
-public class CrawlSalesAmountService extends MybatisBaseService<CrawlSalesAmountMapper, CrawlSalesAmount> {
+public class CrawlSalesAmountService extends ServiceImpl<CrawlSalesAmountMapper, CrawlSalesAmount> {
-
-    @Autowired
-    private CrawlSalesAmountMapper salesAmountMapper;
 
     public ResultBean saveCrawlSalesAmount(CrawlSalesAmountDto dto){
         ResultBean rb = ResultBean.fireFail();
         CrawlSalesAmount amount = new CrawlSalesAmount();
         BeanUtil.copyProperties(dto,amount);
-        int insert = salesAmountMapper.insert(amount);
+        int insert = baseMapper.insert(amount);
         if (insert == 0){
             return rb.setMsg("添加失败");
         }
@@ -53,7 +54,7 @@ public class CrawlSalesAmountService extends MybatisBaseService<CrawlSalesAmount
             }
         }
         IPage<CrawlSalesAmount> page = PagerUtil.queryToPage(pq);
-        IPage<CrawlSalesAmountVo> pagging = salesAmountMapper.selectPageVo(page, qw);
+        IPage<CrawlSalesAmountVo> pagging = baseMapper.selectPageVo(page, qw);
         PagerVo<CrawlSalesAmountVo> pv = PagerUtil.pageToVo(pagging, null);
         List<CrawlSalesAmountVo> records = pv.getRecords();
         for (CrawlSalesAmountVo record : records) {
@@ -64,4 +65,18 @@ public class CrawlSalesAmountService extends MybatisBaseService<CrawlSalesAmount
         }
         return pv;
     }
+
+    public void pullData(String currDay) {
+        String zhjDate = currDay.replace("-","");
+        List<RespSalesAmount> respSalesAmounts = ZhjSpider.crawlSalesAmount(zhjDate);
+        List<CrawlSalesAmount> entityList = new ArrayList<>();
+        for (RespSalesAmount rsa:respSalesAmounts){
+            CrawlSalesAmount csa = new CrawlSalesAmount();
+            BeanUtil.copyProperties(rsa,csa);
+            csa.setDataDate(currDay);
+            entityList.add(csa);
+        }
+        baseMapper.deleteOfDay(currDay);
+        super.saveBatch(entityList);
+    }
 }

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/biz/crawlSalesDishes/CrawlSalesDishes.java

@@ -12,6 +12,8 @@ import io.swagger.annotations.ApiModelProperty;
 public class CrawlSalesDishes extends EntityWithId {
     @ApiModelProperty("数据日期")
     private String dataDate;
+    @ApiModelProperty("门店名称")
+    private String storeName;
     @ApiModelProperty("菜品名称")
     private String dishesName;
     @ApiModelProperty("规格单位")
@@ -29,6 +31,14 @@ public class CrawlSalesDishes extends EntityWithId {
         this.dataDate = dataDate;
     }
 
+    public String getStoreName() {
+        return storeName;
+    }
+
+    public void setStoreName(String storeName) {
+        this.storeName = storeName;
+    }
+
     public String getDishesName() {
         return dishesName;
     }

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/biz/crawlSalesDishes/CrawlSalesDishesMapper.java

@@ -1,7 +1,9 @@
 package com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlSalesDishes;
 
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
 
 import java.util.List;
 
@@ -13,4 +15,7 @@ import java.util.List;
 public interface CrawlSalesDishesMapper extends BaseMapper<CrawlSalesDishes> {
 
     List<CrawlSalesDishes> selectSalesDishesList(CrawlSalesDishesDto dto);
+
+    @Delete("delete from crawl_sales_dishes where dataDate=#{dataDate} ")
+    void deleteOfDay(@Param("dataDate") String currDay);
 }

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/biz/crawlSalesDishes/CrawlSalesDishesService.java

@@ -1,11 +1,17 @@
 package com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlSalesDishes;
 
 import cn.hutool.core.bean.BeanUtil;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.yxt.common.base.service.MybatisBaseService;
 import com.yxt.common.core.result.ResultBean;
+import com.yxt.supervise.dbcenter.zhj.crawl.ZhjSpider;
+import com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlSalesAmount.CrawlSalesAmount;
+import com.yxt.supervise.dbcenter.zhj.crawl.spider.RespSalesAmount;
+import com.yxt.supervise.dbcenter.zhj.crawl.spider.RespSalesDishes;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -13,28 +19,56 @@ import java.util.List;
  * @create 2023-05-09-11:00
  */
 @Service
-public class CrawlSalesDishesService extends MybatisBaseService<CrawlSalesDishesMapper, CrawlSalesDishes> {
+public class CrawlSalesDishesService extends ServiceImpl<CrawlSalesDishesMapper, CrawlSalesDishes> {
 
-    @Autowired
+    public ResultBean save(CrawlSalesDishesDto dto) {
-    private CrawlSalesDishesMapper crawlSalesDishesMapper;
-
-    public ResultBean save(CrawlSalesDishesDto dto){
         ResultBean rb = ResultBean.fireFail();
         CrawlSalesDishes dishes = new CrawlSalesDishes();
-        BeanUtil.copyProperties(dto,dishes);
+        BeanUtil.copyProperties(dto, dishes);
-        int insert = crawlSalesDishesMapper.insert(dishes);
+        int insert = baseMapper.insert(dishes);
-        if (insert == 0){
+        if (insert == 0) {
             return rb.setMsg("添加失败");
         }
         return rb.success().setMsg("添加成功");
     }
 
-    public ResultBean selectSalesDishesList(CrawlSalesDishesDto dto){
+    public ResultBean selectSalesDishesList(CrawlSalesDishesDto dto) {
         ResultBean rb = ResultBean.fireFail();
-        List<CrawlSalesDishes> crawlSalesDishes = crawlSalesDishesMapper.selectSalesDishesList(dto);
+        List<CrawlSalesDishes> crawlSalesDishes = baseMapper.selectSalesDishesList(dto);
-        if (crawlSalesDishes == null || crawlSalesDishes.size() == 0){
+        if (crawlSalesDishes == null || crawlSalesDishes.size() == 0) {
             return rb.setMsg("此门店下没有销售数据");
         }
         return rb.success().setData(crawlSalesDishes);
     }
+
+    public void pullData(String currDay) {
+        String zhjDate = currDay.replace("-", "");
+        List<CrawlSalesDishes> entityList = new ArrayList<>();
+        List<RespSalesDishes> respList1 = ZhjSpider.crawlSalesDishes(zhjDate, "熟溢香振头店");
+        for (RespSalesDishes rsa : respList1) {
+            CrawlSalesDishes csa = new CrawlSalesDishes();
+            BeanUtil.copyProperties(rsa, csa);
+            csa.setDataDate(currDay);
+            csa.setStoreName("熟溢香振头店");
+            entityList.add(csa);
+        }
+        List<RespSalesDishes> respList2 = ZhjSpider.crawlSalesDishes(zhjDate, "馍馍卤振岗店");
+        for (RespSalesDishes rsa : respList2) {
+            CrawlSalesDishes csa = new CrawlSalesDishes();
+            BeanUtil.copyProperties(rsa, csa);
+            csa.setDataDate(currDay);
+            csa.setStoreName("馍馍卤振岗店");
+            entityList.add(csa);
+        }
+        List<RespSalesDishes> respList3 = ZhjSpider.crawlSalesDishes(zhjDate, "馍馍卤雅清街店");
+        for (RespSalesDishes rsa : respList3) {
+            CrawlSalesDishes csa = new CrawlSalesDishes();
+            BeanUtil.copyProperties(rsa, csa);
+            csa.setDataDate(currDay);
+            csa.setStoreName("馍馍卤雅清街店");
+            entityList.add(csa);
+        }
+        baseMapper.deleteOfDay(currDay);
+        super.saveBatch(entityList);
+    }
 }

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/biz/crawlpurchase/CrawlPurchaseMapper.java

@@ -1,7 +1,9 @@
 package com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlpurchase;
 
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
 
 import java.util.List;
 
@@ -12,4 +14,8 @@ import java.util.List;
 @Mapper
 public interface CrawlPurchaseMapper extends BaseMapper<CrawlPurchase> {
     List<CrawlPurchase> selectPurchaseList(CrawlPurchaseDto dto);
+
+
+    @Delete("delete from crawl_purchase where dataDate=#{dataDate} ")
+    void deleteOfDay(@Param("dataDate") String currDay);
 }

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/biz/crawlpurchase/CrawlPurchaseService.java

@@ -1,13 +1,15 @@
 package com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlpurchase;
 
 import cn.hutool.core.bean.BeanUtil;
-import com.yxt.common.base.service.MybatisBaseService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.yxt.common.core.result.ResultBean;
-import com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlSalesAmount.CrawlSalesAmountDto;
+import com.yxt.supervise.dbcenter.zhj.crawl.ZhjSpider;
-import org.springframework.beans.factory.annotation.Autowired;
+import com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlSalesDishes.CrawlSalesDishes;
+import com.yxt.supervise.dbcenter.zhj.crawl.spider.RespPurchase;
+import com.yxt.supervise.dbcenter.zhj.crawl.spider.RespSalesDishes;
 import org.springframework.stereotype.Service;
-import org.springframework.web.bind.annotation.RequestBody;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -15,28 +17,39 @@ import java.util.List;
  * @create 2023-05-09-10:40
  */
 @Service
-public class CrawlPurchaseService extends MybatisBaseService<CrawlPurchaseMapper, CrawlPurchase> {
+public class CrawlPurchaseService extends ServiceImpl<CrawlPurchaseMapper, CrawlPurchase> {
-
-    @Autowired
-    private CrawlPurchaseMapper crawlPurchaseMapper;
 
     public ResultBean save(CrawlPurchaseDto dto){
         ResultBean rb = ResultBean.fireFail();
         CrawlPurchase purchase = new CrawlPurchase();
         BeanUtil.copyProperties(dto,purchase);
-        int insert = crawlPurchaseMapper.insert(purchase);
+        int insert = baseMapper.insert(purchase);
         if (insert == 0){
             return rb.setMsg("添加失败");
         }
         return rb.success().setMsg("添加成功");
     }
 
-    public ResultBean selectPurchaseList(CrawlPurchaseDto dto){
+    public ResultBean<List<CrawlPurchase>> selectPurchaseList(CrawlPurchaseDto dto){
         ResultBean rb = ResultBean.fireFail();
-        List<CrawlPurchase> crawlPurchases = crawlPurchaseMapper.selectPurchaseList(dto);
+        List<CrawlPurchase> crawlPurchases = baseMapper.selectPurchaseList(dto);
         if (crawlPurchases == null || crawlPurchases.size() == 0){
             return rb.setMsg("该店当天没有采购信息");
         }
         return rb.success().setData(crawlPurchases);
     }
+
+    public void pullData(String currDay) {
+        String zhjDate = currDay.replace("-", "");
+        List<RespPurchase> respList = ZhjSpider.crawlPurchase(zhjDate);
+        List<CrawlPurchase> entityList = new ArrayList<>();
+        for (RespPurchase rsa : respList) {
+            CrawlPurchase csa = new CrawlPurchase();
+            BeanUtil.copyProperties(rsa, csa);
+            csa.setDataDate(currDay);
+            entityList.add(csa);
+        }
+        baseMapper.deleteOfDay(currDay);
+        super.saveBatch(entityList);
+    }
 }

yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/test/java/com/yxt/supervise/dbcenter/YxtSuperviseDbcenterApplicationTests.java

@@ -0,0 +1,35 @@
+package com.yxt.supervise.dbcenter;
+
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.thread.ThreadUtil;
+import com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlSalesAmount.CrawlSalesAmountService;
+import com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlSalesDishes.CrawlSalesDishesService;
+import com.yxt.supervise.dbcenter.zhj.crawl.biz.crawlpurchase.CrawlPurchaseService;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import java.util.Date;
+
+@SpringBootTest
+class YxtSuperviseDbcenterApplicationTests {
+
+
+    @Autowired
+    private CrawlSalesAmountService crawlSalesAmountService;
+    @Autowired
+    private CrawlSalesDishesService crawlSalesDishesService;
+    @Autowired
+    private CrawlPurchaseService crawlPurchaseService;
+
+    @Test
+    void contextLoads() {
+        String currDay = "2023-05-05";
+
+        crawlSalesAmountService.pullData(currDay);
+        crawlSalesDishesService.pullData(currDay);
+        crawlPurchaseService.pullData(currDay);
+
+    }
+
+}