算是更新和维护比较多的一个脚本了,不少老哥在用,就更新了几次,从最初的明显的链接bug,到现在可以运行,只是没什么动力,从明显不堪的写法,再回头还是感觉写的比较凌乱,见谅,技术渣!
除了自己用的阿里巴巴商品图片数据采集,应该是这个更新的比较多了!
Python采集工具助手,又拍相册分类页数据信息采集
相册分类页数据采集,分类相册图片及文本采集
没怎么更新代码,只是保证py脚本能跑起来!
如果真要写,还是得重写!
工具说明
又拍相册相册分类页数据采集工具-by huguo002
须正确输入又拍相册某一个分类相册页首页地址,比如:
http://x.yupoo.com/photos/sd6118701/collections/82847
可在又拍相册-全部分类获取某一个分类相册页首页地址
输入地址正确即可采集相册图片及文本数据!
注意地址为相册首页地址,某些相册分类存在分页数据!
BUG 反馈微信:huguo00289
附exe下载地址:
两个版本
又拍云相册图片数据采集助手
链接:
https://pan.baidu.com/s/1UdiGeSVgjCcO70zWEbA5Jg
提取码:
vvqp
又拍云相册分类页数据采集工具
链接:
https://pan.baidu.com/s/1_BM4-egye2QGmaq5B8ag5w
提取码:
fipv
附完整代码:
#又拍图片管家 #https://735539381.x.yupoo.com爬取 #by微信:huguo00289 # -*- coding: UTF-8 -*- import requests,re,os,random,time from lxml import etree from fake_useragent import UserAgent from requests.adapters import HTTPAdapter #引入 HTTPAdapter 库 class Yupoo(): def __init__(self,url): self.ua=UserAgent() self.headers={"User-Agent":self.ua.random} self.zurl="http://x.yupoo.com" self.url=url self.zurl2 = url.split('/collections')[0] # 抓取相册链接 def get_urllists(self,): z = 1 print(f'>>> 正在抓取相册链接:{self.url}') try: response = requests.get(url, headers=self.headers, timeout=10).text time.sleep(1) req = etree.HTML(response) if "pagination__jumpwrap" in response: print(f'{url}存在相册分页链接') num=req.xpath('//form[@class="pagination__jumpwrap"]/span[1]/text()')[0] num=num[1:-1] #print(num) num=int(num) print(f'>>> 共有{num}页分页!') for i in range(1,num+1): numurl=f'{self.url}?page={i}' print(f'>>> 正在采集分页:{numurl}') try: num_response = requests.get(numurl, headers=self.headers, timeout=10).text time.sleep(2) num_req = etree.HTML(num_response) category_urllists = num_req.xpath('//div[@class="showindex__children"]/a[@class="album__main"]/@href') #print(category_urllists) for category_url in category_urllists: if "photos" in category_url: category_url = f'{self.zurl}{category_url}' else: category_url = f'{self.zurl2}{category_url}' try: self.get_img(category_url,z) z = z + 1 except Exception as e: print(f'采集产品相册页链接失败,错误代码:{e}') with open(f'yupoo/spider.txt', 'a+', encoding='utf-8') as f: f.write(f'{category_url}---产品相册页链接采集失败,错误代码:{e}') except Exception as e: print(f'采集相册分页链接失败,错误代码:{e}') with open(f'yupoo/spider.txt', 'a+', encoding='utf-8') as f: f.write(f'{numurl}---相册分页链接采集失败,错误代码:{e}') else: print(f'{url}没有相册分页链接') category_urllists = req.xpath('//div[@class="showindex__children"]/a[@class="album__main"]/@href') for category_url in category_urllists: if "photos" in category_url: category_url = f'{self.zurl}{category_url}' else: category_url = f'{self.zurl2}{category_url}' try: self.get_img(category_url,z) z = z + 1 except Exception as e: print(f'采集产品相册页链接失败,错误代码:{e}') with open(f'yupoo/spider.txt', 'a+', encoding='utf-8') as f: f.write(f'{category_url}---产品相册页链接采集失败,错误代码:{e}') except Exception as e: print(f'采集相册链接失败,错误代码:{e}') with open(f'yupoo/spider.txt', 'a+', encoding='utf-8') as f: f.write(f'{url}---相册链接采集失败,错误代码:{e}') #采集相册详情内容 def get_img(self,url,z): response = requests.get(url,headers=self.headers,timeout=5).text time.sleep(2) req = etree.HTML(response) #采集标题 h2=req.xpath('//h2/span[@class="showalbumheader__gallerytitle"]/text()')[0] h2= re.sub(r'[\|\/\<\>\:\*\?\\\"]', "_", h2) # 剔除不合法字符 print(h2) #创建目录 os.makedirs(f'yupoo/{z}{h2}/',exist_ok=True) #采集文字详情 try: text=req.xpath('//div[@class="showalbumheader__gallerysubtitle htmlwrap__main"]/text()')[0] except Exception as e: print(f'无详情文字内容!') text=[] print(text) if text!=[]: texts='%s%s%s'%(h2,'\n',text) with open(f'yupoo/{z}{h2}/{h2}.txt','w',encoding='utf-8') as f: f.write(texts) print(f'{z}{h2}.txt 文档保存成功!') #采集图片 imgurls=req.xpath('//div[@class="showalbum__children image__main"]/div[@class="image__imagewrap"]/img/@data-origin-src') i=1 headers2 = { 'referer': url, 'user-agent': self.ua.random, } for imgurl in imgurls: imgurl=f'https:{imgurl}' print(imgurl) if "jpeg" in imgurl: imgname=f'{i}{imgurl[-5:]}' elif "jgp" in imgurl or "png" in imgurl or "gif" in imgurl: imgname = f'{i}{imgurl[-4:]}' else: imgname = f'{i}{imgurl[-4:]}' print(imgname) s = requests.session() s.mount('http://', HTTPAdapter(max_retries=3)) s.mount('https://', HTTPAdapter(max_retries=3)) try: r=requests.get(imgurl,headers=headers2,timeout=8) with open(f'yupoo/{z}{h2}/{imgname}','wb') as f: f.write(r.content) print(f'下载{imgname}图片成功!') time.sleep(1) except requests.exceptions.RequestException as e: print(f'下载{imgurl}图片失败,错误代码:{e}') with open(f'yupoo/{z}{h2}/spider.txt','a+',encoding='utf-8') as f: f.write(f'访问相册{url}---{imgurl}---下载失败,错误代码:{e}') i=i+1 print(f'采集 {z}{h2} 图片成功!') if __name__ == '__main__': print('\n========================工具说明========================') print('又拍相册相册分类页数据采集工具-by huguo002') print('须正确输入又拍相册某一个分类相册页首页地址,比如:') print('http://x.yupoo.com/photos/sd6118701/collections/82847') print('可在又拍相册-全部分类获取某一个分类相册页首页地址') print('输入地址正确即可采集相册图片及文本数据!') print('注意地址为相册首页地址,某些相册分类存在分页数据!') print('================= BUG 反馈微信:huguo00289 ============\n') time.sleep(8) url=input("请输入要爬取的又拍相册分类相册页首页地址:") print(f'爬虫启动中,请稍后......') spider=Yupoo(url) print(f'相册所有链接采集中,时间可能稍长,请稍后......') spider.get_urllists() print(f'采集完毕,采集{url}图片成功!') print(f'10后自动结束程序,退出!') time.sleep(5)