第 4 章：数据持久化——将战利品存入仓库 (Data Persistence)¶

import csv

# 假设 movies_data 是第 3 章中提取的电影字典列表
movies_data = [
    {"排名": 1, "中文名": "肖申克的救赎", "评分": 9.7, "短评": "希望让人自由。"},
    {"排名": 2, "中文名": "霸王别姬", "评分": 9.6, "短评": "风华绝代。"},
    # ... 更多数据
]

# 指定 CSV 文件路径
csv_file = "douban_top250.csv"

# newline="" 参数防止 Windows 下出现多余空行
# encoding="utf-8-sig" 确保 Excel 能正确识别中文（BOM 头）
with open(csv_file, "w", newline="", encoding="utf-8-sig") as f:
    # 从第一条数据中提取表头（字典的键）
    fieldnames = movies_data[0].keys()

    # 创建 DictWriter 对象
    # DictWriter 会根据 fieldnames 自动将字典映射到 CSV 列
    writer = csv.DictWriter(f, fieldnames=fieldnames)

    # 写入表头行
    writer.writeheader()

    # 逐行写入数据
    for movie in movies_data:
        writer.writerow(movie)

print(f"✅ 数据已导出到 {csv_file}")

# csv 模块自动处理特殊字符
movie = {"导演": "弗兰克·德拉邦特, 蒂姆·罗宾斯"}  # 包含逗号
writer.writerow(movie)
# 输出: "弗兰克·德拉邦特, 蒂姆·罗宾斯"  ← 自动加引号

import pandas as pd

# 将字典列表直接转换为 DataFrame
# DataFrame 是 pandas 的核心数据结构，类似 Excel 表格
df = pd.DataFrame(movies_data)

# 按评分降序排列
df_sorted = df.sort_values(by="评分", ascending=False)

# 导出为 CSV
# index=False 表示不导出行号
df_sorted.to_csv("douban_top250_sorted.csv", index=False, encoding="utf-8-sig")

# 查看数据概览
print(f"数据形状: {df.shape}")  # (25, 7) 表示 25 行 7 列
print(f"平均评分: {df['评分'].mean():.2f}")
print(f"最高评分: {df['评分'].max()}")
print(f"最低评分: {df['评分'].min()}")

import json

json_file = "douban_top250.json"

# ensure_ascii=False 确保中文不被转义为 \uXXXX
# indent=2 使输出格式化，便于人工阅读
with open(json_file, "w", encoding="utf-8") as f:
    json.dump(movies_data, f, ensure_ascii=False, indent=2)

print(f"✅ 数据已导出到 {json_file}")

import csv
import os

csv_file = "douban_top250_all.csv"
file_exists = os.path.isfile(csv_file)

# 以追加模式打开文件
with open(csv_file, "a", newline="", encoding="utf-8-sig") as f:
    fieldnames = ["排名", "中文名", "评分", "短评"]
    writer = csv.DictWriter(f, fieldnames=fieldnames)

    # 只在文件不存在时写入表头（避免重复表头）
    if not file_exists:
        writer.writeheader()

    # 追加新数据
    for movie in new_batch_of_movies:
        writer.writerow(movie)

import json

json_file = "douban_top250_all.json"

# 尝试读取已有数据
try:
    with open(json_file, "r", encoding="utf-8") as f:
        existing_data = json.load(f)
except (FileNotFoundError, json.JSONDecodeError):
    existing_data = []

# 合并新旧数据
existing_data.extend(new_batch_of_movies)

# 写回文件
with open(json_file, "w", encoding="utf-8") as f:
    json.dump(existing_data, f, ensure_ascii=False, indent=2)

"""
电影数据抓取与存储模块
功能：抓取豆瓣电影 Top 250 首页数据，并导出为 CSV 和 JSON
"""
import requests
import csv
import json
from bs4 import BeautifulSoup


def fetch_movies():
    """抓取豆瓣电影 Top 250 首页数据"""
    url = "https://movie.douban.com/top250"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    }

    response = requests.get(url, headers=headers, timeout=10)
    response.encoding = response.apparent_encoding
    soup = BeautifulSoup(response.text, "lxml")

    movie_items = soup.find_all("div", class_="item")
    movies = []

    for item in movie_items:
        rank = item.find("em").get_text()

        title_spans = item.find_all("span", class_="title")
        chinese_title = title_spans[0].get_text()
        english_title = title_spans[1].get_text().replace("\xa0/\xa0", "").strip() if len(title_spans) > 1 else "无"

        rating = item.find("span", class_="rating_num").get_text()

        rating_people_text = item.find("div", class_="star").find_all("span")[-1].get_text()
        rating_people = rating_people_text.replace("人评价", "")

        quote_tag = item.find("span", class_="inq")
        quote = quote_tag.get_text() if quote_tag else "无短评"

        info_p = item.find("p", class_="")
        director_info = info_p.get_text().strip().replace("\xa0", " ")

        link = item.find("a")["href"]

        movies.append({
            "排名": int(rank),
            "中文名": chinese_title,
            "英文名": english_title,
            "评分": float(rating),
            "评价人数": rating_people,
            "短评": quote,
            "详情": director_info,
            "链接": link,
        })

    return movies


def save_to_csv(movies, filename="douban_top250.csv"):
    """将电影数据保存为 CSV 文件"""
    with open(filename, "w", newline="", encoding="utf-8-sig") as f:
        fieldnames = movies[0].keys()
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(movies)
    print(f"✅ CSV 已保存: {filename} ({len(movies)} 条记录)")


def save_to_json(movies, filename="douban_top250.json"):
    """将电影数据保存为 JSON 文件"""
    with open(filename, "w", encoding="utf-8") as f:
        json.dump(movies, f, ensure_ascii=False, indent=2)
    print(f"✅ JSON 已保存: {filename} ({len(movies)} 条记录)")


if __name__ == "__main__":
    print("正在抓取豆瓣电影 Top 250 首页...")
    movies = fetch_movies()

    if movies:
        save_to_csv(movies)
        save_to_json(movies)
        print(f"\n📊 数据概览:")
        print(f"   共抓取 {len(movies)} 部电影")
        print(f"   评分最高: {movies[0]['中文名']} ({movies[0]['评分']})")
        print(f"   评分最低: {movies[-1]['中文名']} ({movies[-1]['评分']})")
    else:
        print("❌ 未抓取到任何数据，请检查网络或请求头配置。")