【更新】Python采集工具助手,又拍相册分类页数据信息采集

算是更新和维护比较多的一个脚本了,不少老哥在用,就更新了几次,从最初的明显的链接bug,到现在可以运行,只是没什么动力,从明显不堪的写法,再回头还是感觉写的比较凌乱,见谅,技术渣!

除了自己用的阿里巴巴商品图片数据采集,应该是这个更新的比较多了!

pic_001.jpg

Python采集工具助手,又拍相册分类页数据信息采集

相册分类页数据采集,分类相册图片及文本采集

pic_002.jpg

没怎么更新代码,只是保证py脚本能跑起来!

如果真要写,还是得重写!


工具说明

又拍相册相册分类页数据采集工具-by huguo002

须正确输入又拍相册某一个分类相册页首页地址,比如:

http://x.yupoo.com/photos/sd6118701/collections/82847

可在又拍相册-全部分类获取某一个分类相册页首页地址

输入地址正确即可采集相册图片及文本数据!

注意地址为相册首页地址,某些相册分类存在分页数据!

BUG 反馈微信:huguo00289 


附exe下载地址:

两个版本


又拍云相册图片数据采集助手

链接: 

https://pan.baidu.com/s/1UdiGeSVgjCcO70zWEbA5Jg 

提取码: 

vvqp


又拍云相册分类页数据采集工具

链接: 

https://pan.baidu.com/s/1_BM4-egye2QGmaq5B8ag5w 

提取码: 

fipv


附完整代码:

#又拍图片管家
#https://735539381.x.yupoo.com爬取
#by微信:huguo00289

# -*- coding: UTF-8 -*-
import requests,re,os,random,time
from lxml import etree
from fake_useragent import UserAgent
from requests.adapters import HTTPAdapter #引入 HTTPAdapter 库

class Yupoo():
    def __init__(self,url):
        self.ua=UserAgent()
        self.headers={"User-Agent":self.ua.random}
        self.zurl="http://x.yupoo.com"
        self.url=url
        self.zurl2 = url.split('/collections')[0]


    # 抓取相册链接
    def get_urllists(self,):
        z = 1
        print(f'>>> 正在抓取相册链接:{self.url}')
        try:
            response = requests.get(url, headers=self.headers, timeout=10).text
            time.sleep(1)
            req = etree.HTML(response)
            if "pagination__jumpwrap" in response:
                print(f'{url}存在相册分页链接')
                num=req.xpath('//form[@class="pagination__jumpwrap"]/span[1]/text()')[0]
                num=num[1:-1]
                #print(num)
                num=int(num)
                print(f'>>> 共有{num}页分页!')
                for i in range(1,num+1):
                    numurl=f'{self.url}?page={i}'
                    print(f'>>> 正在采集分页:{numurl}')
                    try:
                        num_response = requests.get(numurl, headers=self.headers, timeout=10).text
                        time.sleep(2)
                        num_req = etree.HTML(num_response)
                        category_urllists = num_req.xpath('//div[@class="showindex__children"]/a[@class="album__main"]/@href')
                        #print(category_urllists)
                        for category_url in category_urllists:
                            if "photos" in category_url:
                                category_url = f'{self.zurl}{category_url}'
                            else:
                                category_url = f'{self.zurl2}{category_url}'
                            try:
                                self.get_img(category_url,z)
                                z = z + 1
                            except Exception as e:
                                print(f'采集产品相册页链接失败,错误代码:{e}')
                                with open(f'yupoo/spider.txt', 'a+', encoding='utf-8') as f:
                                    f.write(f'{category_url}---产品相册页链接采集失败,错误代码:{e}')

                    except Exception as e:
                        print(f'采集相册分页链接失败,错误代码:{e}')
                        with open(f'yupoo/spider.txt', 'a+', encoding='utf-8') as f:
                            f.write(f'{numurl}---相册分页链接采集失败,错误代码:{e}')

            else:
                print(f'{url}没有相册分页链接')
                category_urllists = req.xpath('//div[@class="showindex__children"]/a[@class="album__main"]/@href')
                for category_url in category_urllists:
                    if "photos" in category_url:
                        category_url = f'{self.zurl}{category_url}'
                    else:
                        category_url = f'{self.zurl2}{category_url}'
                    try:
                        self.get_img(category_url,z)
                        z = z + 1
                    except Exception as e:
                        print(f'采集产品相册页链接失败,错误代码:{e}')
                        with open(f'yupoo/spider.txt', 'a+', encoding='utf-8') as f:
                            f.write(f'{category_url}---产品相册页链接采集失败,错误代码:{e}')

        except Exception as e:
            print(f'采集相册链接失败,错误代码:{e}')
            with open(f'yupoo/spider.txt', 'a+', encoding='utf-8') as f:
                f.write(f'{url}---相册链接采集失败,错误代码:{e}')


    #采集相册详情内容
    def get_img(self,url,z):
        response = requests.get(url,headers=self.headers,timeout=5).text
        time.sleep(2)
        req = etree.HTML(response)
        #采集标题
        h2=req.xpath('//h2/span[@class="showalbumheader__gallerytitle"]/text()')[0]
        h2= re.sub(r'[\|\/\<\>\:\*\?\\\"]', "_", h2)  # 剔除不合法字符
        print(h2)
        #创建目录
        os.makedirs(f'yupoo/{z}{h2}/',exist_ok=True)
        #采集文字详情
        try:
            text=req.xpath('//div[@class="showalbumheader__gallerysubtitle htmlwrap__main"]/text()')[0]
        except Exception as e:
            print(f'无详情文字内容!')
            text=[]
        print(text)
        if text!=[]:
            texts='%s%s%s'%(h2,'\n',text)
            with open(f'yupoo/{z}{h2}/{h2}.txt','w',encoding='utf-8') as f:
                f.write(texts)
                print(f'{z}{h2}.txt 文档保存成功!')

        #采集图片
        imgurls=req.xpath('//div[@class="showalbum__children image__main"]/div[@class="image__imagewrap"]/img/@data-origin-src')
        i=1
        headers2 = {
            'referer': url,
            'user-agent': self.ua.random,
        }
        for imgurl in imgurls:
            imgurl=f'https:{imgurl}'
            print(imgurl)
            if "jpeg" in imgurl:
                imgname=f'{i}{imgurl[-5:]}'
            elif "jgp" in imgurl or "png" in imgurl or "gif" in imgurl:
                imgname = f'{i}{imgurl[-4:]}'
            else:
                imgname = f'{i}{imgurl[-4:]}'
            print(imgname)
            s = requests.session()
            s.mount('http://', HTTPAdapter(max_retries=3))
            s.mount('https://', HTTPAdapter(max_retries=3))
            try:
                r=requests.get(imgurl,headers=headers2,timeout=8)
                with open(f'yupoo/{z}{h2}/{imgname}','wb') as f:
                    f.write(r.content)
                    print(f'下载{imgname}图片成功!')
                    time.sleep(1)
            except requests.exceptions.RequestException as e:
                print(f'下载{imgurl}图片失败,错误代码:{e}')
                with open(f'yupoo/{z}{h2}/spider.txt','a+',encoding='utf-8') as f:
                    f.write(f'访问相册{url}---{imgurl}---下载失败,错误代码:{e}')
            i=i+1
        print(f'采集 {z}{h2} 图片成功!')




if __name__ == '__main__':
    print('\n========================工具说明========================')
    print('又拍相册相册分类页数据采集工具-by huguo002')
    print('须正确输入又拍相册某一个分类相册页首页地址,比如:')
    print('http://x.yupoo.com/photos/sd6118701/collections/82847')
    print('可在又拍相册-全部分类获取某一个分类相册页首页地址')
    print('输入地址正确即可采集相册图片及文本数据!')
    print('注意地址为相册首页地址,某些相册分类存在分页数据!')
    print('================= BUG 反馈微信:huguo00289 ============\n')
    time.sleep(8)
    url=input("请输入要爬取的又拍相册分类相册页首页地址:")
    print(f'爬虫启动中,请稍后......')
    spider=Yupoo(url)
    print(f'相册所有链接采集中,时间可能稍长,请稍后......')
    spider.get_urllists()

    print(f'采集完毕,采集{url}图片成功!')
    print(f'10后自动结束程序,退出!')
    time.sleep(5)