提交任意时间

.idea/vcs.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file

crawler.py

@@ -47,20 +47,20 @@ def SaveCompanyData(name,types,intro):
 #启动爬虫
 def CrawlerLaunch():
     while True:
-        randomtime = random.randint(3, 100)
-        time.sleep(randomtime)
         now = datetime.datetime.now()
-
-        print(str(now)+":启动")
+        randomtime = random.randint(3, 120)
+        print(str(now) + ":启动等待时间"+str(randomtime))
+        time.sleep(randomtime)
         crawler_baidu.CrawlerBaidu()
         #crawler_boss.CrawlerBoss()
         # 在进程中执行的任务
 
 
 def Log(dataS):
-    file = open(str(datetime.date.today()) + ".txt", "a+")
-    file.write(dataS+"\n")
-    file.close()
+    with open(str(datetime.date.today()) + ".txt", "a+") as f:
+        f.write(dataS + "\n")
+        f.close()
+
 
 if __name__ == "__main__":
     p = multiprocessing.Process(target=CrawlerLaunch)

crawler_baidu.py

@@ -12,7 +12,7 @@ def CrawlerBaidu():
 
 #爬取百度的数据
 def GetBaiduCompany(playwright: Playwright) -> None:
-    browser = playwright.chromium.launch(headless=True)
+    browser = playwright.chromium.launch(headless=False)
     context = browser.new_context(viewport={"width": 800, "height": 600})
     page = context.new_page()
     #获取需要爬取数据的公司

crawler_boss.py

@@ -13,11 +13,6 @@ def spider_company(page):
     company_detail_el_count = company_detail_el.count()
     print("company_detail_el.count():", company_detail_el_count)
     my_obj = {'intro': ""}
-    post_data = {
-        "name": company_nameO,
-        "content": json.dumps(my_obj),
-        "type": 2
-    }
     if company_detail_el_count > 0:
         company_detail_el.first.click()
         page.wait_for_timeout(1000)
@@ -38,14 +33,12 @@ def spider_company(page):
 
 def GetBossCompany(p: Playwright) -> None:
     browser = p.chromium.launch(headless=True)
-    context = browser.new_context()
+    context = browser.new_context(viewport={"width": 800, "height": 600})
     js = """
     Object.defineProperties(navigator, {webdriver:{get:()=>undefined}});
     """
     page = context.new_page()
     page.add_init_script(js)
-    # spider = BossSpider()
-    # spider.spider_company(page)
     spider_company(page)
 
     context.close()