import requests

url = https://translate.google.cn/

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',}

# requests.get 使用GET方法
response = requests.get(url, headers=headers)
response.encoding = 'utf-8'
# response是一个Response对象，我们想要的页面信息在response.text里面
data = response.text
print(data)  # data是一个含有页面信息的字符串

import requests
import json
from lxml import etree


class Dang():
    def __init__(self):
        '''
        初始化全局url地址，设置headers
        '''
        self.start_url = "http://e.dangdang.com/list-XS2-comment-0-{}.html"  # 这里设置了一个'{}'是为了后面可以改变页码
        self.headers = {
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
                    }
        self.page = 0

    def get_url(self, url):
        '''
        获取指定url地址的html字符串
        '''
        response = requests.get(url, headers = self.headers)
        html_str = response.content.decode()
        element = etree.HTML(html_str)
        return element

    def parse(self, element):
        '''
        使用xpath从html字符串中解析出目标内容
        '''
        # // 的意思是选取当前节点下符合条件的全部节点并作为一个 list 对象返回，而且以 '/' 开头表示从根节点开始选取
        element_list = element.xpath("//div[@class='book book_list clearfix']")

        book_list = []
        for book in element_list:
            # '.' 表示从当前节点开始选取，下面这个语句的意思是选取当前节点下class属性为'title'的所有<div>节点中的文本
            title_list = book.xpath(".//div[@class='title']/text()") 
            author_list = book.xpath(".//div[@class='author']/text()")
            href_list = book.xpath(".//a/@href")
            price_list = book.xpath(".//span[@class='now']/text()")

        for i in range(len(title_list)):
            # 将上面每一本书的内容按照（页码，编号，书名，作者，链接，价格）的顺序保存
            if i == 0:
                self.page += 1
            book = [self.page, i+1, title_list[i], author_list[i], href_list[i], price_list[i]]

            book_list.append(book)

        return book_list

    def write_to_file(self, book_list, page):
        '''
        将爬取的内容写入到本地文件
        '''
        with open("dangdang.txt", "a", encoding = "utf-8") as f:
            for book in book_list:
                # 这里使用json.dumps() 将Python对象解析为可写入文件的json字符串
                f.write(json.dumps(book, ensure_ascii = False))
                f.write("\n")
        f.close()
        print("successfully wrote page {}".format(page))

    def run(self):
        num = 1
        while num < 11: # 爬取前10页的内容
            # 1.初始url地址
            url = self.start_url.format(num)
            # 2.获取响应
            element = self.get_url(url)
            # 3.xpath解析
            book_list = self.parse(element)
            # 4.保存数据
            self.write_to_file(book_list, num)
            num +=1

dangdang = Dang()
dangdang.run()

from bs4 import BeautifulSoup