Untitled5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e939a064-7110-4fea-97c0-5c2d1e46083a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "成功爬取数据：http://www.tianqihoubao.com/aqi/beijing-202101.html\n",
      "成功爬取数据：http://www.tianqihoubao.com/aqi/beijing-202102.html\n",
      "成功爬取数据：http://www.tianqihoubao.com/aqi/beijing-202103.html\n",
      "成功爬取数据：http://www.tianqihoubao.com/aqi/beijing-202104.html\n",
      "成功爬取数据：http://www.tianqihoubao.com/aqi/beijing-202105.html\n",
      "成功爬取数据：http://www.tianqihoubao.com/aqi/beijing-202106.html\n",
      "成功爬取数据：http://www.tianqihoubao.com/aqi/beijing-202107.html\n",
      "成功爬取数据：http://www.tianqihoubao.com/aqi/beijing-202108.html\n",
      "成功爬取数据：http://www.tianqihoubao.com/aqi/beijing-202109.html\n",
      "成功爬取数据：http://www.tianqihoubao.com/aqi/beijing-202110.html\n"
     ]
    },
    {
     "ename": "PermissionError",
     "evalue": "[Errno 13] Permission denied: 'weather3.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mPermissionError\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[3], line 67\u001b[0m\n\u001b[0;32m     65\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m u \u001b[38;5;129;01min\u001b[39;00m urls:     \u001b[38;5;66;03m# 在网址中循环\u001b[39;00m\n\u001b[0;32m     66\u001b[0m     time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m2\u001b[39m)  \u001b[38;5;66;03m# 每次爬取休息 2 秒，以免太过频繁的请求\u001b[39;00m\n\u001b[1;32m---> 67\u001b[0m     crawler(u, save_file)\n",
      "Cell \u001b[1;32mIn[3], line 59\u001b[0m, in \u001b[0;36mcrawler\u001b[1;34m(url, save_path)\u001b[0m\n\u001b[0;32m     57\u001b[0m response \u001b[38;5;241m=\u001b[39m get_response(url)    \u001b[38;5;66;03m# 请求数据\u001b[39;00m\n\u001b[0;32m     58\u001b[0m results \u001b[38;5;241m=\u001b[39m parse_data(response)  \u001b[38;5;66;03m# 解析数据\u001b[39;00m\n\u001b[1;32m---> 59\u001b[0m save_data(results, save_path)   \u001b[38;5;66;03m# 存储数据\u001b[39;00m\n\u001b[0;32m     60\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m成功爬取数据：\u001b[39m\u001b[38;5;132;01m{\u001b[39;00murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n",
      "Cell \u001b[1;32mIn[3], line 38\u001b[0m, in \u001b[0;36msave_data\u001b[1;34m(weather_list, save_path)\u001b[0m\n\u001b[0;32m     37\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msave_data\u001b[39m(weather_list, save_path):\n\u001b[1;32m---> 38\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(save_path, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124ma\u001b[39m\u001b[38;5;124m'\u001b[39m, newline\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m, encoding\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m fp:\n\u001b[0;32m     39\u001b[0m         csv_header \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m空气质量指数\u001b[39m\u001b[38;5;124m'\u001b[39m]  \u001b[38;5;66;03m# 设置表头，即列名\u001b[39;00m\n\u001b[0;32m     40\u001b[0m         csv_writer \u001b[38;5;241m=\u001b[39m csv\u001b[38;5;241m.\u001b[39mDictWriter(fp, csv_header)\n",
      "File \u001b[1;32mD:\\anaconda\\Lib\\site-packages\\IPython\\core\\interactiveshell.py:310\u001b[0m, in \u001b[0;36m_modified_open\u001b[1;34m(file, *args, **kwargs)\u001b[0m\n\u001b[0;32m    303\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m file \u001b[38;5;129;01min\u001b[39;00m {\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m}:\n\u001b[0;32m    304\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m    305\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIPython won\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt let you open fd=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfile\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m by default \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    306\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mas it is likely to crash IPython. If you know what you are doing, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    307\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124myou can use builtins\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m open.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    308\u001b[0m     )\n\u001b[1;32m--> 310\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m io_open(file, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "\u001b[1;31mPermissionError\u001b[0m: [Errno 13] Permission denied: 'weather3.csv'"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "# Author: W.Y.\n",
    "# Email: wangyingchn@outlook.com\n",
    "# Date: 2022/4/4\n",
    "\n",
    "# 导入模块\n",
    "import csv  # 用于存储数据\n",
    "import time  # 用于时间间隔避免过频繁的请求\n",
    "import requests  # 用于请求数据\n",
    "from bs4 import BeautifulSoup  # 用于解析数据\n",
    "\n",
    "# 请求数据\n",
    "def get_response(url):\n",
    "    # 伪装一下，让服务器以为是正常浏览，而不是爬虫。\n",
    "    # 静态网页通常反爬不严格，所以只要通过 User-Agent 伪装成浏览器即可。\n",
    "    headers = {\n",
    "        \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \\\n",
    "        (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36\"\n",
    "    }\n",
    "    # 请求数据。使用 get 方法请求数据\n",
    "    response = requests.get(url, headers=headers)\n",
    "    return response\n",
    "\n",
    "# 解析数据\n",
    "def parse_data(response):\n",
    "    soup = BeautifulSoup(response.text, \"html.parser\")  # 由于是通过 html 格式存储的，所以用 “html.parser” 进行解析\n",
    "    data_table = soup.find('table', attrs={'width':'620px','border':'0','class':'b','cellpadding':'1','cellspacing':'1'}).find_all(\"tr\")  # 找到包裹表内容的 ul 标签，找到里面所有的 li 标签\n",
    "    weather_list = []  # 构造空列表以存储数据\n",
    "    for tr in data_table[1:]:  # 循环获取每行的数据（li 标签）\n",
    "        th_list = tr.find_all('td')  # 获取每行的每个数据 （li 标签下的 div 标签）\n",
    "        weather = {\n",
    "            '空气质量指数': th_list[2].get_text(),  # 获取第一个 div 标签中的内容，命名为 “date”\n",
    "        }   # 每行数据存储在一个字典中\n",
    "        weather_list.append(weather)  # 所有行的数据存入一个列表中\n",
    "    return weather_list\n",
    "# 储存数据\n",
    "def save_data(weather_list, save_path):\n",
    "    with open(save_path, 'a', newline='', encoding='utf-8') as fp:\n",
    "        csv_header = ['空气质量指数']  # 设置表头，即列名\n",
    "        csv_writer = csv.DictWriter(fp, csv_header)\n",
    "        if fp.tell() == 0:\n",
    "            csv_writer.writeheader()  # 如果文件不存在，则写入表头；如果文件已经存在，则直接追加数据不再次写入表头\n",
    "        csv_writer.writerows(weather_list)  # 写入数据\n",
    "\n",
    "# 构造网址\n",
    "def generate_urls():\n",
    "    url_pattern = 'http://www.tianqihoubao.com/aqi/beijing-2021{}.html'  # 网址的基本结构，有变化的两个部分用 {} 替代，后面循环补充\n",
    "    months = [str(x).zfill(2) for x in range(1, 13)]  # 生成月份列表，zfill 函数补充两位数\n",
    "    month_list = [str(month)for month in months]  # 年月循环拼在一起\n",
    "    url_list = []  # 空列表用于存储所有网址\n",
    "    for m in month_list:  # 再循环时间\n",
    "            url_list.append(url_pattern.format(m))  # 通过 format 函数生成网址\n",
    "    return url_list\n",
    "\n",
    "# 定义爬取函数\n",
    "def crawler(url, save_path):\n",
    "    response = get_response(url)    # 请求数据\n",
    "    results = parse_data(response)  # 解析数据\n",
    "    save_data(results, save_path)   # 存储数据\n",
    "    print(f'成功爬取数据：{url}')\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    urls = generate_urls()     # 构造所有网址\n",
    "    save_file = 'weather3.csv'  # 保存数据的文件路径\n",
    "    for u in urls:     # 在网址中循环\n",
    "        time.sleep(2)  # 每次爬取休息 2 秒，以免太过频繁的请求\n",
    "        crawler(u, save_file)  # 进行爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "defab936-12e3-42a9-b075-e8fa66483355",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}