JavaScript动态渲染界面爬取-Selenium实战

爬取的网页为：https://spa2.scrape.center，里面的内容都是通过Ajax渲染出来的，在分析xhr时候发现url里面有token参数，所有我们使用selenium自动化工具来爬取JavaScript渲染的界面。

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.support.ui import WebDriverWait
import logging
from selenium.webdriver.support import expected_conditions
import re
import json
from os import makedirs
from os.path import exists# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s: %(message)s')
# 基本url
url = "https://spa2.scrape.center/page/{page}"
# selenium初始化
browser = webdriver.Chrome()
# 显式等待初始化
wait = WebDriverWait(browser, 10)
book_url = list()# 目录设置
RESULTS_DIR = 'results'
exists(RESULTS_DIR) or makedirs(RESULTS_DIR)
# 任意异常
class ScraperError(Exception):pass# 获取书本URL
def PageDetail(URL):browser.get(URL)try:all_element = wait.until(expected_conditions.presence_of_all_elements_located((By.CSS_SELECTOR, ".el-card .name")))return all_elementexcept TimeoutException:logging.info("Time error happen in %s while finding the href", URL)# 获取书本信息
def GetDetail(book_list):try:for book in book_list:browser.get(book)URL = browser.current_urlbook_name = wait.until(expected_conditions.presence_of_element_located((By.CLASS_NAME, "m-b-sm"))).textcategories = [elements.text for elements in wait.until(expected_conditions.presence_of_all_elements_located((By.CSS_SELECTOR, ".categories button span")))]content = wait.until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, ".item .drama p[data-v-f7128f80]"))).textdetail = {"URL": URL,"book_name": book_name,"categories": categories,"content": content}SaveDetail(detail)except TimeoutException:logging.info("Time error happen in %s while finding the book detail", browser.current_url)# JSON文件保存
def SaveDetail(detail):cleaned_name = re.sub(r'[\/:*?"<>|]', '_', detail.get("book_name"))detail["book_name"] = cleaned_namedata_path = f'{RESULTS_DIR}/{cleaned_name}.json'logging.info("Saving Book %s...", cleaned_name)try:json.dump(detail, open(data_path, 'w', encoding='utf-8'),ensure_ascii=False, indent=2)logging.info("Saving Book %s over", cleaned_name)except ScraperError as e:logging.info("Some error happen in %s while saving the book detail", cleaned_name)# 主函数
def main():try:for page in range(1, 11):for each_page in PageDetail(url.format(page= page)):book_url.append(each_page.get_attribute("href"))GetDetail(book_url)except ScraperError as e:logging.info("An abnormal position has occurred")finally:browser.close()if __name__ == "__main__":main()