diff --git a/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/ZhjSpider.java b/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/ZhjSpider.java index d207b4dc..cea514d2 100644 --- a/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/ZhjSpider.java +++ b/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/ZhjSpider.java @@ -3,6 +3,7 @@ package com.yxt.supervise.dbcenter.zhj.crawl; import cn.hutool.http.HttpRequest; import cn.hutool.json.JSONUtil; import com.yxt.supervise.dbcenter.zhj.crawl.spider.*; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +65,7 @@ public class ZhjSpider { } public static void main(String[] args) { - List s = ZhjSpider.crawlSalesAmount("20230508"); + List s = ZhjSpider.crawlSalesAmount("20230508"); // List s = ZhjSpider.crawlSalesDishes("20230509", "熟溢香振头店"); // List s = ZhjSpider.crawlPurchase("20230508"); System.out.println(s); @@ -171,9 +172,18 @@ public class ZhjSpider { } public static ZhjResp dopost(Map paramMap) { - HttpRequest request = ZhjSpiderConfig.fillHeaders(HttpRequest.post(url)); + HttpRequest post = HttpRequest.post(url); +// post.enableDefaultCookie(); +// post.disableCookie(); + HttpRequest request = ZhjSpiderConfig.fillHeaders(post); String result = request.form(paramMap).execute().body(); - L.info("param:{}######result:{}", paramMap.toString(), result); + L.info("param:{}", paramMap.toString()); + L.info("######result-111:{}", result); + if (StringUtils.isBlank(result)) { + ZhjSpiderConfig.switchVer(); + result = ZhjSpiderConfig.fillHeaders(post).form(paramMap).execute().body(); + L.info("######result-222:{}", result); + } ZhjResp bean = JSONUtil.toBean(result, ZhjResp.class); return bean; } diff --git a/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/ZhjSpiderConfig.java b/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/ZhjSpiderConfig.java index 9a00704c..a072f49c 100644 --- a/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/ZhjSpiderConfig.java +++ b/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/main/java/com/yxt/supervise/dbcenter/zhj/crawl/ZhjSpiderConfig.java @@ -23,11 +23,26 @@ public class ZhjSpiderConfig { return request; } + private static String v_flag = "1"; + private static void initHeaders() { CACHE_HEADERS = new HashMap<>(); - CACHE_HEADERS.put("cookie", "cna=QZ74GyioHjsCAWpxs/DlZflP; qbi_locale=zh-CN; x_login_pk=19b6a50a88ee4c8cbeacee4420345262; xlly_s=1; qbi_version=1; qbi_redirected=1; csrf_token=f66e628f-d681-4be5-855e-56e77886e002; isg=BIqKYOxE2qEZjVZawN-NAZ2Y23Asew7V-iD6yBTCI11JxyuB_Q3P5fXx1zMbN4Zt"); +// CACHE_HEADERS.put("cookie", "cna=QZ74GyioHjsCAWpxs/DlZflP; qbi_locale=zh-CN; x_login_pk=19b6a50a88ee4c8cbeacee4420345262; xlly_s=1; qbi_version=1; qbi_redirected=1; csrf_token=f66e628f-d681-4be5-855e-56e77886e002; isg=BIqKYOxE2qEZjVZawN-NAZ2Y23Asew7V-iD6yBTCI11JxyuB_Q3P5fXx1zMbN4Zt"); +// CACHE_HEADERS.put("cookie", "cna=QZ74GyioHjsCAWpxs/DlZflP; qbi_locale=zh-CN; x_login_pk=19b6a50a88ee4c8cbeacee4420345262; csrf_token=f66e628f-d681-4be5-855e-56e77886e002; qbi_version=2; xlly_s=1; isg=BLe3V-4In-iHGRsxfXgo-thHRqsBfIven4934wljCQdwuN36EEhuLkRemhjmUGNW"); + CACHE_HEADERS.put("cookie", "cna=QZ74GyioHjsCAWpxs/DlZflP; qbi_locale=zh-CN; x_login_pk=19b6a50a88ee4c8cbeacee4420345262; csrf_token=f66e628f-d681-4be5-855e-56e77886e002; qbi_version=" + v_flag + "; xlly_s=1; isg=BLe3V-4In-iHGRsxfXgo-thHRqsBfIven4934wljCQdwuN36EEhuLkRemhjmUGNW"); +// CACHE_HEADERS.put("x-csrf-token", "f66e628f-d681-4be5-855e-56e77886e002"); CACHE_HEADERS.put("x-csrf-token", "f66e628f-d681-4be5-855e-56e77886e002"); - CACHE_HEADERS.put("x-gw-referer", "https://bi.aliyuncs.com/token3rd/dashboard/view/pc.htm?pageId=85c4af04-1910-4005-8e47-742af1acb713&accessToken=d8c021cd40ad94b2d500d12229b57c3e&dd_orientation=auto&qbi_version_param=1"); +// CACHE_HEADERS.put("x-gw-referer", "https://bi.aliyuncs.com/token3rd/dashboard/view/pc.htm?pageId=85c4af04-1910-4005-8e47-742af1acb713&accessToken=d8c021cd40ad94b2d500d12229b57c3e&dd_orientation=auto&qbi_version_param=1"); +// CACHE_HEADERS.put("x-gw-referer", "https://bi.aliyuncs.com/token3rd/dashboard/view/pc.htm?pageId=85c4af04-1910-4005-8e47-742af1acb713&accessToken=d8c021cd40ad94b2d500d12229b57c3e&dd_orientation=auto&qbi_version_param=2"); + CACHE_HEADERS.put("x-gw-referer", "https://bi.aliyuncs.com/token3rd/dashboard/view/pc.htm?pageId=85c4af04-1910-4005-8e47-742af1acb713&accessToken=d8c021cd40ad94b2d500d12229b57c3e&dd_orientation=auto&qbi_version_param=" + v_flag); // CACHE_HEADERS.put("x-gw-referer", "https://bi.aliyuncs.com/token3rd/dashboard/view/pc.htm?pageId=85c4af04-1910-4005-8e47-742af1acb713&accessToken=d8c021cd40ad94b2d500d12229b57c3e&dd_orientation=auto"); } + + public static void switchVer() { + if ("1".equals(v_flag)) + v_flag = "2"; + else + v_flag = "1"; + initHeaders(); + } } diff --git a/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/test/java/com/yxt/supervise/dbcenter/YxtSuperviseDbcenterApplicationTests.java b/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/test/java/com/yxt/supervise/dbcenter/YxtSuperviseDbcenterApplicationTests.java index 216cc39c..91c093af 100644 --- a/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/test/java/com/yxt/supervise/dbcenter/YxtSuperviseDbcenterApplicationTests.java +++ b/yxt-supervise-dbcenter/yxt-supervise-dbcenter-biz/src/test/java/com/yxt/supervise/dbcenter/YxtSuperviseDbcenterApplicationTests.java @@ -24,7 +24,7 @@ class YxtSuperviseDbcenterApplicationTests { @Test void contextLoads() { - String currDay = "2023-05-17"; + String currDay = "2023-05-18"; // crawlSalesAmountService.pullData(currDay); // crawlSalesDishesService.pullData(currDay);