创新实训（十二）爬虫

爬虫入门

1. requests+bs4
2. selenium+geckodriver
3. scrapy框架

Mac安装geckodriver
brew install geckodriver
设置配置文件~/.bash_profile文件
export path

import requests
from bs4 import BeautifulSoup

url = "https://b2c.csair.com/B2C40/newTrips/static/main/page/booking/index.html?t=S&c1=BJS&c2=SHA&d1=2021-04-20&at=1&ct=0&it=0&b1=PEK-PKX&b2=SHA-PVG"

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36'}


r = requests.get(url, headers=headers)
r.encoding = 'UTF-8'

soup = BeautifulSoup(r.text, "html.parser")
title = soup.find("title").text

print(title)

from selenium import webdriver

driver = webdriver.Firefox(executable_path=r'geckodriver')
url = "https://b2c.csair.com/B2C40/newTrips/static/main/page/booking/index.html?t=S&c1=BJS&c2=SHA&d1=2021-04-20&at=1&ct=0&it=0&b1=PEK-PKX&b2=SHA-PVG"
driver.get(url)

print(driver.page_source)
comment = driver.find_element_by_css_selector('div.zls-flplace')
content = comment.find_element_by_class_name('zls-flplace')
print(content.text)