提交

2023-08-29.txt

+久久王食品国际有限公司:写入成功

crawler.py

@@ -4,7 +4,7 @@ import time
 import json
 import crawler_baidu
 import crawler_boss
-
+import random
 import requests
 
 #获取需要爬数据的企业 百度1
@@ -47,17 +47,20 @@ def SaveCompanyData(name,types,intro):
 #启动爬虫
 def CrawlerLaunch():
     while True:
-        time.sleep(3)
+        randomtime = random.randint(3, 100)
+        time.sleep(randomtime)
         now = datetime.datetime.now()
 
         print(str(now)+":启动")
-        # crawler_baidu.CrawlerBaidu()
-        crawler_boss.CrawlerBoss()
+        crawler_baidu.CrawlerBaidu()
+        #crawler_boss.CrawlerBoss()
         # 在进程中执行的任务
 
-def Log():
-    file = open(str(datetime.date.today()) + ".txt", "a")
 
+def Log(dataS):
+    file = open(str(datetime.date.today()) + ".txt", "a+")
+    file.write(dataS+"\n")
+    file.close()
 
 if __name__ == "__main__":
     p = multiprocessing.Process(target=CrawlerLaunch)

crawler_baidu.py

@@ -5,8 +5,6 @@ import json
 import datetime
 import crawler
 
-
-
 #爬虫启动
 def CrawlerBaidu():
     with sync_playwright() as playwright:
@@ -25,14 +23,9 @@ def GetBaiduCompany(playwright: Playwright) -> None:
         intro = re.sub(r'\[[\d-]+\]', '', str(all))
         name2 = crawler.SaveCompanyData(name,1,intro)
         if name2 != "" and intro != None:
-            file = open(str(datetime.date.today()) + ".txt", "a")
-
-            file.write(name2+":写入成功\n")
-            file.close()
+            crawler.Log(name2+":写入成功")
         else:
-            file = open("example.txt", "a")
-            file.write(name2+":写入失败\n")
-            file.close()
+            crawler.Log(name2 + ":写入失败")
 
         # ---------------------
         context.close()

crawler_boss.py

@@ -37,7 +37,7 @@ def spider_company(page):
     name2 = crawler.SaveCompanyData(company_nameO, 1, json.dumps(my_obj))
 
 def GetBossCompany(p: Playwright) -> None:
-    browser = p.chromium.launch(headless=False)
+    browser = p.chromium.launch(headless=True)
     context = browser.new_context()
     js = """
     Object.defineProperties(navigator, {webdriver:{get:()=>undefined}});