svg矢量图批量下载翻页物品爬虫程序代码QZQ1

import requests import json import os import re import time # 新增：用于控制请求延迟 from lxml import etree # 1. 关键：从浏览器复制完整的 Cookie 和 Headers headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36 Edg/150.0.0.0', 'Referer': 'https://freesvglab.com/zh/svg/line-art-rooster-farm', # 关键：完整的Cookie字符串（从浏览器F12复制） 'Cookie': 'cf_clearance=4HDa3sVuwMymqY0Jo9YUBFmSKT8qa4w7eXaBJveQUB8-1785585211-1.2.1.1-HNzZBo.2P6oXd.AcezEbmv63.DaYC4YWvsA0iX_PrVu5MiyoAkPcuDmtkyXwWXb49EvP5QbvhJo3sFzpxRDvNOez2dZlVnkwOjI_I4TatyPHqHWGSCpVc.ENcwzHFd115j1Sn4XXEWRqYbmrv3lGQfS4khPbynrWle3GFPtGL9IV_SDZjMWyN1xTpYoDM97q9elLpyeFlMFJqY5uK7060rpjEyfnYAQ62JNTgLxOxo0hu5vxM0nqdrZk9edLV9MayTRPzxAByKxw0yPmIdM0CMaxm9y4jxfkiq0VTSJH2njvrefkwcdqYj7I3XrDvYc49vT5._KCMRx3ln9ZxX.Zx0k8VNc7bRO.Qatu3pVlqCk; _ga=GA1.1.1678275870.1785585329; _ga_P3K7XNCWBJ=GS2.1.s1785585329$o1$g1$t1785585461$j35$l0$h0' } # 创建一个文件夹用于存放视频片段，取名为"物品" if not os.path.exists("物品"): os.mkdir("物品") for page in range(1,6): print(f'--------- 正在爬取第{page}页的内容 ----------') # if page==1: # url = 'https://freesvglab.com/zh/category/objects/page/1/' # 请求地址 # else: # url = f'https://freesvglab.com/zh/category/objects/page/{page}.html' # 请求地址 # 第36行修正 - 统一使用 /page/{page}/ if page == 1: url = 'https://freesvglab.com/zh/category/objects/' # 第1页 else: url = f'https://freesvglab.com/zh/category/objects/page/{page}/' # 第2页开始 #https://freesvglab.com/zh/category/objects/page/2 # url = f'https://freesvglab.com/zh/category/objects' response = requests.get(url=url, headers=headers).text # print(response.text) # html_data=re.findall('"url":"(.*?)"}',response.text) # print(html_data) # 使用XPath解析页面 page_html = etree.HTML(response) # 提取歌曲列表信息 # id_list = page_html.xpath('//html/body/main/div[2]/a[*]') # id_list = page_html.xpath('//html/body/main/div[2]/a[*]/@href') id_list = page_html.xpath('//html/body/main/div[2]/a/@href') # print(id_list) # 解析歌曲列表信息，并逐个提交下载任务到线程池 for n in id_list: i = "https://freesvglab.com" + n # print(i) # /html/body/main/div[2]/a[2] # https://freesvglab.com/zh/svg/logo-lantern res = requests.get(url = i, headers=headers).text # print(res) # 使用XPath解析页面 page_html = etree.HTML(res) # # 提取歌曲列表信息 # # id_list = page_html.xpath('//html/body/main/div[2]/a[*]') # d = page_html.xpath('//html/body/main/article/div[2]/div[2]/a[2]/@href')[0] # print(d) n = page_html.xpath('//html/body/main/article/div[2]/div[1]/h1/text()')[0] # print(n) # break u = "https://freesvglab.com" + d # //html/body/main/article/div[2]/div[2]/a[1] # //html/body/main/article/div[2]/div[2]/a[2]/@href # https://freesvglab.com/images/vectors/logo/lantern.svg # 判断文件是否已经存在 # break # file_path = os.path.join("物品", n + ".svg") file_path = os.path.join("物品", f"{n}_{int(time.time())}.svg") response = requests.get(url=u, headers=headers).content with open(file_path, 'wb') as f: f.write(response) print(n, '下载完成') # //html/body/main/article/div[2]/div[1]/h1/text()