爬虫 | 豆瓣读书、当当网图书排行榜

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
# -*- coding: utf-8 -*-#
"""
@Author  :   haauleon
@Contact :   753494552@qq.com
"""
import requests
from lxml import etree
from enum import  Enum


headers={
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
}

# 当当网页面元素枚举
class DDWPageEle(Enum):
    ele_names = '//div[2]/ul/li/div[3]/a/@title'
    ele_comments = '//div[2]/ul/li/div[4]/a/text()'
    ele_fivestars = '//div[2]/ul/li/div[7]/span/text()'


# 豆瓣读书网页面元素枚举
class DBPageEle(Enum):
    ele_names = '//table/tr/td[2]/div[1]/a/@title'
    ele_authors = '//table/tr/td[2]/p[1]/text()'
    ele_comments = '//table/tr/td[2]/div[2]/span[3]/text()'



class DangDangWang:

    def __init__(self):
        self.headers = headers
        self.names = None
        self.comments = None
        self.fivestars = None

    def get_book_info(self, url):
        """获取当当网图书排行榜"""
        res = requests.get(url, headers=self.headers)
        res.encoding = res.apparent_encoding  # 将乱码进行编码
        html = etree.HTML(res.text)
        for book in html:
            self.names = book.xpath(DDWPageEle.ele_names.value)
            self.comments = book.xpath(DDWPageEle.ele_comments.value)
            self.fivestars = book.xpath(DDWPageEle.ele_fivestars.value)

        for name, comment, fivestar in zip(self.names, self.comments, self.fivestars):
            yield name, comment, fivestar


class DouBan:

    def __init__(self):
        self.headers = headers
        self.names = None
        self.authors = None
        self.comments = None

    def get_book_info(self, url):
        """获取豆瓣读书排行榜"""
        res = requests.get(url, headers=self.headers)
        html = etree.HTML(res.text)
        for book in html:
            self.names = book.xpath(DBPageEle.ele_names.value)
            self.authors = book.xpath(DBPageEle.ele_authors.value)
            self.comments = book.xpath(DBPageEle.ele_comments.value)

        for name, author, comment in zip(self.names, self.authors, self.comments):
            yield name.strip(), author.split('/')[0].strip(), comment.replace('(', '').replace(')', '').strip()


def run_ddw_spider():
    """执行爬取当当网图书数据"""
    urls = ['http://bang.dangdang.com/books/fivestars/01.00.00.00.00.00-recent30-0-0-1-{}'.format(i) for i in range(1,26)]
    ddw = DangDangWang()
    rank = 1
    for url in urls:
        print(url)
        for name, comment, fivestar in ddw.get_book_info(url):
            print(rank, name, comment, fivestar)
            rank += 1


def run_db_spider():
    """执行爬取豆瓣图书数据"""
    urls = ['https://book.douban.com/top250?start={}'.format(i*25) for i in range(10)]
    db = DouBan()
    rank = 1
    for url in urls:
        print(url)
        for name, author, comment in db.get_book_info(url):
            print(rank, name, author, comment)
            rank += 1


if __name__=='__main__':
    run_ddw_spider()
    print('-'*50)
    run_db_spider()