Python 爬虫接口实战：翻页取完再累加（CH-002）

1
"""CH-002：答案 = 全部电影「评分总人数」之和"""
2
import re
3
import time
4

5
import requests
6

7
BASE = "https://spider.jsnote.top"
8
API_URL = f"{BASE}/api/movies"
9
PAGE_SIZE = 12
10

11
session = requests.Session()
12
session.headers.update({
13
    "Accept": "application/json, text/plain, */*",
14
    "Accept-Language": "zh-CN,zh;q=0.9",
15
    "Referer": f"{BASE}/movies?challenge=CH-002",
16
    "User-Agent": (
17
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
18
        "(KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
19
    ),
20
})
21

22
# 数据是按账号派生的，所以要带登录态：
23
# 浏览器登录后 F12 -> Application -> Cookies -> 复制 session 的值
24
session.cookies.set("session", "<把浏览器里的 session cookie 粘到这里>",
25
                    domain="spider.jsnote.top")
26

27

28
def rating_count(movie):
29
    """优先用字段，字段缺失/为 0 时回头从 '3235402人评价' 里抠数字。"""
30
    count = movie.get("rating_count") or 0
31
    if count:
32
        return int(count)
33
    match = re.search(r"([\d,]+)\s*人评价", movie.get("rating_count_text", ""))
34
    return int(match.group(1).replace(",", "")) if match else 0
35

36

37
def main():
38
    page = 1
39
    total_movies = None
40
    total_count = 0
41
    movies = 0
42

43
    while True:
44
        response = session.get(
45
            API_URL, params={"page": page, "page_size": PAGE_SIZE}, timeout=10
46
        )
47
        response.raise_for_status()
48
        payload = response.json()
49

50
        items = payload.get("items", [])
51
        if not items:
52
            break
53

54
        total_movies = payload.get("total_movies", total_movies)
55
        for movie in items:
56
            total_count += rating_count(movie)
57
            movies += 1
58

59
        print(f"第 {page:>2} 页：{len(items)} 条，累计 {movies} 条，"
60
              f"评价人数小计 {total_count}")
61

62
        if total_movies and movies >= total_movies:
63
            break
64
        page += 1
65
        time.sleep(0.5)          # 别把靶场当压力测试
66

67
    print(f"\n共 {movies} 部电影，评价人数总和：{total_count}")
68

69

70
if __name__ == "__main__":
71
    main()