Преглед изворни кода

添加SpiderSession类,负责Session相关操作

HuangYiwei пре 5 година
родитељ
комит
d6912d9e65
1 измењених фајлова са 92 додато и 0 уклоњено
  1. 92 0
      jd_spider_requests.py

+ 92 - 0
jd_spider_requests.py

@@ -16,6 +16,98 @@ from concurrent.futures import ProcessPoolExecutor
 from exception import SKException
 
 
+class SpiderSession:
+    """
+    Session相关操作
+    """
+    def __init__(self):
+        self.cookies_dir_path = "./cookies/"
+        self.is_login = False
+
+        self.session = self.__init_session()
+        self.load_cookies_from_local()
+
+    def __init_session(self):
+        session = requests.session()
+        session.headers = {"User-Agent": global_config.getRaw('config', 'DEFAULT_USER_AGENT'),
+                           "Accept": "text/html,application/xhtml+xml,application/xml;"
+                                     "q=0.9,image/webp,image/apng,*/*;"
+                                     "q=0.8,application/signed-exchange;"
+                                     "v=b3",
+                           "Connection": "keep-alive"}
+        return session
+
+    def get_session(self):
+        """
+        获取当前Session
+        :return:
+        """
+        return self.session
+
+    def get_cookies(self):
+        """
+        获取当前Cookies
+        :return:
+        """
+        return self.get_session().cookies
+
+    def set_cookies(self, cookies):
+        self.session.cookies.update(cookies)
+
+    def update_login_status(self):
+        self.is_login = self.__validate_cookies()
+
+    def load_cookies_from_local(self):
+        """
+        从本地加载Cookie
+        :return:
+        """
+        cookies_file = ''
+        if not os.path.exists(self.cookies_dir_path):
+            return False
+        for name in os.listdir(self.cookies_dir_path):
+            if name.endswith(self.cookies_dir_path):
+                cookies_file = '{}{}'.format(self.cookies_dir_path, name)
+                break
+        if cookies_file == '':
+            return False
+        with open(cookies_file, 'rb') as f:
+            local_cookies = pickle.load(f)
+        self.set_cookies(local_cookies)
+        self.update_login_status()
+
+    def save_cookies_to_local(self, cookie_file_name):
+        """
+        保存Cookie到本地
+        :param cookie_file_name: 存放Cookie的文件名称
+        :return:
+        """
+        cookies_file = '{}{}'.format(self.cookies_dir_path, cookie_file_name)
+        directory = os.path.dirname(cookies_file)
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+        with open(cookies_file, 'wb') as f:
+            pickle.dump(self.get_cookies(), f)
+
+    def __validate_cookies(self):
+        """
+        验证cookies是否有效(是否登陆)
+        通过访问用户订单列表页进行判断:若未登录,将会重定向到登陆页面。
+        :return: cookies是否有效 True/False
+        """
+        url = 'https://order.jd.com/center/list.action'
+        payload = {
+            'rid': str(int(time.time() * 1000)),
+        }
+        try:
+            resp = self.session.get(url=url, params=payload, allow_redirects=False)
+            if resp.status_code == requests.codes.OK:
+                return True
+        except Exception as e:
+            logger.error("验证cookies是否有效发生异常", e)
+        return False
+
+
 class JdSeckill(object):
     def __init__(self):
         # 初始化信息