下厨房抓取

最新推荐文章于 2023-11-20 16:39:45 发布
最新推荐文章于 2023-11-20 16:39:45 发布
阅读量688 收藏
点赞数
CC 4.0 BY-SA版权
分类专栏： 爬虫
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/weixin_42357472/article/details/87183252

下厨房整体还是比较容易，就是有一个302链接跳转需要判断下即可

import requests
from lxml import etree
import re
import json
import csv
import pandas as pd
import hashlib

product_lists=[]
def down_load(url):
    headers1 = {
        # "Cookie":"SINAGLOBAL=7238757845138.87.1528291392417; UOR=,,spr_web_360_hao360_weibo_t001; login_sid_t=bd5a4abe734c091249cdce71379c0348; cross_origin_proto=SSL; Ugrow-G0=e66b2e50a7e7f417f6cc12eec600f517; TC-V5-G0=866fef700b11606a930f0b3297300d95; _s_tentry=-; Apache=685802145012.8082.1542780237180; ULV=1542780237187:19:3:1:685802145012.8082.1542780237180:1541462062210; TC-Page-G0=cdcf495cbaea129529aa606e7629fea7; WBtopGlobal_register_version=18608f873d5d88f2; SSOLoginState=1542781061; wvr=6; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W50QEC8VdzuOHjXwxjEGser5JpX5K2hUgL.Fo-feo.ceKe4S0M2dJLoIpjLxKqLBoqL1-qLxKqLB.eLB-2LxKqL1KMLB.2t; ALF=1574324177; SCF=ApgoQqG5luyu67rkHic6LidzChLHTIe5EQZgRnsuPrfkK57iJqk723zd_GSb5ZMq2jbGlYvGXkZ6LbJj5PpY6zI.; SUB=_2A2528WQBDeRhGeNL6VsX8S3FzDuIHXVVh9LJrDV8PUNbmtAKLVnXkW9NSQ30mXwLfrcwH1SRYaTHBUXB4ipbEQrL; SUHB=02MvCTyTmQYvsK; un=18514476337; YF-V5-G0=a5a6106293f9aeef5e34a2e71f04fae4; wb_view_log_5529613977=1920*10801",

        # "Cache-Control":"max-age=0",
        "Connection": "keep-alive",
        "Cookie": "bid=YTzZGjuP; gr_user_id=97f998e4-aa93-486a-99f8-854c784f6bcd; BAIDU_SSP_lcr=https://www.baidu.com/link?url=Pwq6pphyjxk3oQRJg05jjUGiwtLjL7FBVFdqA8oFqpZoXc0hIxm9p0sa3fyWRSSs&wd=&eqid=c9c1816400092dd0000000035c218bdd; Hm_lvt_ecd4feb5c351cc02583045a5813b5142=1545271479,1545702369; __utmc=177678124; __utmz=177678124.1545702370.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; gr_session_id_8187ff886f0929da=3a10677b-5117-45e8-9de4-d80fc87ffc01; gr_session_id_da48e7b9eb89482489897fc1e45e98b6=3f128e3a-5f5c-42fd-9c45-e0e1ddeb97b2; _ga=GA1.2.377109310.1545271479; _gid=GA1.2.1629322193.1545716866; __utma=177678124.377109310.1545271479.1545702370.1545717020.3; __utmt=1; gr_session_id_8187ff886f0929da_3a10677b-5117-45e8-9de4-d80fc87ffc01=true; Hm_lpvt_ecd4feb5c351cc02583045a5813b5142=1545717163; __utmb=177678124.4.10.1545717020",
        "Host": "www.xiachufang.com",
        # "Referer":"https://weibo.com/",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",

        # "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
        # "Accept-Encoding":"gzip, deflate, br",
        # "Accept-Language":"zh-CN,zh;q=0.9",
    }

    html = requests.get(url=url,headers=headers1,allow_redirects=False)
    print(html.status_code)
    # print(html.headers["location"])
    if html.status_code == 302:
        new_id_url="http://www.xiachufang.com"+html.headers["location"]
        print(new_id_url)
        new_html=requests.get(url=new_id_url,headers=headers1).text
        return new_id_url,etree.HTML(new_html)
    else:
        print("++++++++++++++++")
        print(url)
        # print(requests.get(url=url,headers=headers1).text)
        return url,etree.HTML(requests.get(url=url,headers=headers1).text)

    # return (new_html,new_id_url)


def down_load1(url):
    headers2 = {
        # "Cookie":"SINAGLOBAL=7238757845138.87.1528291392417; UOR=,,spr_web_360_hao360_weibo_t001; login_sid_t=bd5a4abe734c091249cdce71379c0348; cross_origin_proto=SSL; Ugrow-G0=e66b2e50a7e7f417f6cc12eec600f517; TC-V5-G0=866fef700b11606a930f0b3297300d95; _s_tentry=-; Apache=685802145012.8082.1542780237180; ULV=1542780237187:19:3:1:685802145012.8082.1542780237180:1541462062210; TC-Page-G0=cdcf495cbaea129529aa606e7629fea7; WBtopGlobal_register_version=18608f873d5d88f2; SSOLoginState=1542781061; wvr=6; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W50QEC8VdzuOHjXwxjEGser5JpX5K2hUgL.Fo-feo.ceKe4S0M2dJLoIpjLxKqLBoqL1-qLxKqLB.eLB-2LxKqL1KMLB.2t; ALF=1574324177; SCF=ApgoQqG5luyu67rkHic6LidzChLHTIe5EQZgRnsuPrfkK57iJqk723zd_GSb5ZMq2jbGlYvGXkZ6LbJj5PpY6zI.; SUB=_2A2528WQBDeRhGeNL6VsX8S3FzDuIHXVVh9LJrDV8PUNbmtAKLVnXkW9NSQ30mXwLfrcwH1SRYaTHBUXB4ipbEQrL; SUHB=02MvCTyTmQYvsK; un=18514476337; YF-V5-G0=a5a6106293f9aeef5e34a2e71f04fae4; wb_view_log_5529613977=1920*10801",

        # "Cache-Control":"max-age=0",
        "Connection": "keep-alive",
        "Cookie": "bid=YTzZGjuP; gr_user_id=97f998e4-aa93-486a-99f8-854c784f6bcd; BAIDU_SSP_lcr=https://www.baidu.com/link?url=Pwq6pphyjxk3oQRJg05jjUGiwtLjL7FBVFdqA8oFqpZoXc0hIxm9p0sa3fyWRSSs&wd=&eqid=c9c1816400092dd0000000035c218bdd; Hm_lvt_ecd4feb5c351cc02583045a5813b5142=1545271479,1545702369; __utmc=177678124; __utmz=177678124.1545702370.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; gr_session_id_8187ff886f0929da=3a10677b-5117-45e8-9de4-d80fc87ffc01; gr_session_id_da48e7b9eb89482489897fc1e45e98b6=3f128e3a-5f5c-42fd-9c45-e0e1ddeb97b2; _ga=GA1.2.377109310.1545271479; _gid=GA1.2.1629322193.1545716866; __utma=177678124.377109310.1545271479.1545702370.1545717020.3; __utmt=1; gr_session_id_8187ff886f0929da_3a10677b-5117-45e8-9de4-d80fc87ffc01=true; Hm_lpvt_ecd4feb5c351cc02583045a5813b5142=1545717163; __utmb=177678124.4.10.1545717020",
        "Host": "www.xiachufang.com",
        # "Referer":"https://weibo.com/",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",

        # "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
        # "Accept-Encoding":"gzip, deflate, br",
        # "Accept-Language":"zh-CN,zh;q=0.9",
    }

    html = requests.get(url=url,headers=headers2).text
    # return (new_html,new_id_url)
    return etree.HTML(html)


def down_load2(url):
    headers3 = {
        # "Cookie":"SINAGLOBAL=7238757845138.87.1528291392417; UOR=,,spr_web_360_hao360_weibo_t001; login_sid_t=bd5a4abe734c091249cdce71379c0348; cross_origin_proto=SSL; Ugrow-G0=e66b2e50a7e7f417f6cc12eec600f517; TC-V5-G0=866fef700b11606a930f0b3297300d95; _s_tentry=-; Apache=685802145012.8082.1542780237180; ULV=1542780237187:19:3:1:685802145012.8082.1542780237180:1541462062210; TC-Page-G0=cdcf495cbaea129529aa606e7629fea7; WBtopGlobal_register_version=18608f873d5d88f2; SSOLoginState=1542781061; wvr=6; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W50QEC8VdzuOHjXwxjEGser5JpX5K2hUgL.Fo-feo.ceKe4S0M2dJLoIpjLxKqLBoqL1-qLxKqLB.eLB-2LxKqL1KMLB.2t; ALF=1574324177; SCF=ApgoQqG5luyu67rkHic6LidzChLHTIe5EQZgRnsuPrfkK57iJqk723zd_GSb5ZMq2jbGlYvGXkZ6LbJj5PpY6zI.; SUB=_2A2528WQBDeRhGeNL6VsX8S3FzDuIHXVVh9LJrDV8PUNbmtAKLVnXkW9NSQ30mXwLfrcwH1SRYaTHBUXB4ipbEQrL; SUHB=02MvCTyTmQYvsK; un=18514476337; YF-V5-G0=a5a6106293f9aeef5e34a2e71f04fae4; wb_view_log_5529613977=1920*10801",

        # "Cache-Control":"max-age=0",
        "Connection": "keep-alive",
        "Cookie": "bid=YTzZGjuP; gr_user_id=97f998e4-aa93-486a-99f8-854c784f6bcd; BAIDU_SSP_lcr=https://www.baidu.com/link?url=Pwq6pphyjxk3oQRJg05jjUGiwtLjL7FBVFdqA8oFqpZoXc0hIxm9p0sa3fyWRSSs&wd=&eqid=c9c1816400092dd0000000035c218bdd; Hm_lvt_ecd4feb5c351cc02583045a5813b5142=1545271479,1545702369; __utmc=177678124; __utmz=177678124.1545702370.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; gr_session_id_8187ff886f0929da=3a10677b-5117-45e8-9de4-d80fc87ffc01; gr_session_id_da48e7b9eb89482489897fc1e45e98b6=3f128e3a-5f5c-42fd-9c45-e0e1ddeb97b2; _ga=GA1.2.377109310.1545271479; _gid=GA1.2.1629322193.1545716866; __utma=177678124.377109310.1545271479.1545702370.1545717020.3; __utmt=1; gr_session_id_8187ff886f0929da_3a10677b-5117-45e8-9de4-d80fc87ffc01=true; Hm_lpvt_ecd4feb5c351cc02583045a5813b5142=1545717163; __utmb=177678124.4.10.1545717020",
        "Host": "www.xiachufang.com",
        # "Referer":"https://weibo.com/",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",

        # "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
        # "Accept-Encoding":"gzip, deflate, br",
        # "Accept-Language":"zh-CN,zh;q=0.9",
    }

    html = requests.get(url=url,headers=headers3).text
    # return (new_html,new_id_url)
    return etree.HTML(html)




word_pd=pd.read_csv(r"C:\Users\Lavector\Desktop\百事小红书\ele1.csv",engine='python',header=None).values.tolist()
print(word_pd)
# for m in range(1):
for m in range(1,len(word_pd)):
    key_word=word_pd[m][0]
    print(key_word)

    url1="http://www.xiachufang.com/search/?keyword={}".format(key_word)

    try:
# key_word="酸奶"
#         url1 = "http://www.xiachufang.com/search/?keyword={}".format(key_word)
        product_html_all=down_load(url1)
        product_html=product_html_all[1]
        print(len(product_html.xpath('//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li')))
        for w in range(len(product_html.xpath('//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li'))):
            # for i in range(1):
            print("**************")
            product_url="http://www.xiachufang.com"+product_html.xpath('//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="name"]/a/@href'.format(w+1))[0]
            print(product_url)
            product_name=product_html.xpath('//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="name"]/a/text()'.format(w+1))[0].strip()
            if product_html.xpath('//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[1]/text()'.format(w+1)):

                product_star=product_html.xpath('//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[1]/text()'.format(w+1))[0]
            else:
                product_star="暂无"
            if product_html.xpath('//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[@class="bold score"]/text()'.format(w+1)):
                product_make=product_html.xpath('//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[@class="bold score"]/text()'.format(w+1))[0]
            else:
                product_make=0
            product_author=product_html.xpath('//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="author"]/a[1]/text()'.format(w+1))[0]
            product_author_url="http://www.xiachufang.com"+product_html.xpath('//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="author"]/a[1]/@href'.format(w+1))[0]
            person_html=down_load2(product_author_url)
            person_info1=person_html.xpath('//div[@class="gray-font"]/div[1]')[0]
            person_info=person_info1.xpath('string(.)').strip().replace(" ","").replace("\n","")

            print(person_info)
            print(product_html.xpath('//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="name"]/a/@href'.format(w+1))[0])
            print(product_name)
            detil_html=down_load1(product_url)[1]
            product_save=detil_html.xpath('//div[@class="pv"]/text()')[0]
            product_content1=detil_html.xpath('//div[contains(@class,"recipe-show")]')[0]
            product_content=product_content1.xpath('string(.)').strip().replace(" ","").replace("\n","")
            product_time=detil_html.xpath('//div[@class="time"]/span/text()')[0]
            product_meke_all=detil_html.xpath('//div[contains(@class,"cooked")]/span[@class="number"]/text()')[0]
            print(product_meke_all)
            product_lists.append(
                [key_word, product_author, person_info, product_name, product_url, product_content, product_star,
                 product_meke_all, product_make, product_save, product_time])
        print(product_html[0])
        print("++++++++")
        if "category" in product_html_all[0]:
            for k in range(2):
                url2 = product_html_all[0]+"?page={}".format(str(k+2))
                product_html2 = down_load2(url2)
                for p in range(len(product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li'))):
                    # for i in range(1):
                    print("**************")
                    product_url = "http://www.xiachufang.com" + product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="name"]/a/@href'.format(
                            p + 1))[0]
                    print(product_url)
                    product_name = product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="name"]/a/text()'.format(
                            p + 1))[0].strip()
                    if product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[1]/text()'.format(
                            p + 1)):
                        product_star = product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[1]/text()'.format(
                            p + 1))[0]
                    else:
                        product_star="暂无"
                    if product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[@class="bold score"]/text()'.format(
                            p + 1)):
                        product_make = product_html2.xpath(
                            '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[@class="bold score"]/text()'.format(
                                p + 1))[0]
                    else:
                        product_make=0
                    product_author = product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="author"]/a[1]/text()'.format(
                            p + 1))[0]
                    product_author_url = "http://www.xiachufang.com" + product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="author"]/a[1]/@href'.format(
                            p + 1))[0]
                    person_html = down_load2(product_author_url)
                    person_info1 = person_html.xpath('//div[@class="gray-font"]/div[1]')[0]
                    person_info = person_info1.xpath('string(.)').strip().replace(" ", "").replace("\n", "")

                    print(person_info)
                    print(product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="name"]/a/@href'.format(
                            p + 1))[0])
                    print(product_name)
                    detil_html = down_load1(product_url)[1]
                    product_save = detil_html.xpath('//div[@class="pv"]/text()')[0]
                    product_content1 = detil_html.xpath('//div[contains(@class,"recipe-show")]')[0]
                    product_content = product_content1.xpath('string(.)').strip().replace(" ", "").replace("\n", "")
                    product_time = detil_html.xpath('//div[@class="time"]/span/text()')[0]
                    product_meke_all = \
                    detil_html.xpath('//div[contains(@class,"cooked")]/span[@class="number"]/text()')[0]
                    print(product_meke_all)
                    product_lists.append(
                        [key_word, product_author, person_info, product_name, product_url, product_content,
                         product_star,
                         product_meke_all, product_make, product_save, product_time])
        else:
            for k in range(2):
                url3 = product_html_all[0] + "&page={}".format(str(k + 2))
                product_html2 = down_load2(url3)
                for p in range(len(product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li'))):
                    # for i in range(1):
                    print("**************")
                    product_url = "http://www.xiachufang.com" + product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="name"]/a/@href'.format(
                            p + 1))[0]
                    print(product_url)
                    product_name = product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="name"]/a/text()'.format(
                            p + 1))[0].strip()
                    if product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[1]/text()'.format(
                            p + 1)):
                        product_star = product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[1]/text()'.format(
                            p + 1))[0]
                    else:
                        product_star="暂无"
                    if product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[@class="bold score"]/text()'.format(
                            p + 1)):
                        product_make = product_html2.xpath(
                            '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="stats"]/span[@class="bold score"]/text()'.format(
                                p + 1))[0]
                    else:
                        product_make=0
                    product_author = product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="author"]/a[1]/text()'.format(
                            p + 1))[0]
                    product_author_url = "http://www.xiachufang.com" + product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="author"]/a[1]/@href'.format(
                            p + 1))[0]
                    person_html = down_load2(product_author_url)
                    person_info1 = person_html.xpath('//div[@class="gray-font"]/div[1]')[0]
                    person_info = person_info1.xpath('string(.)').strip().replace(" ", "").replace("\n", "")

                    print(person_info)
                    print(product_html2.xpath(
                        '//div[contains(@class,"pure-u-3-4")]//div[@class="normal-recipe-list"]//ul[@class="list"]/li[{}]//p[@class="name"]/a/@href'.format(
                            p + 1))[0])
                    print(product_name)
                    detil_html = down_load1(product_url)[1]
                    product_save = detil_html.xpath('//div[@class="pv"]/text()')[0]
                    product_content1 = detil_html.xpath('//div[contains(@class,"recipe-show")]')[0]
                    product_content = product_content1.xpath('string(.)').strip().replace(" ", "").replace("\n", "")
                    product_time = detil_html.xpath('//div[@class="time"]/span/text()')[0]
                    product_meke_all = \
                        detil_html.xpath('//div[contains(@class,"cooked")]/span[@class="number"]/text()')[0]
                    print(product_meke_all)
                    product_lists.append(
                        [key_word, product_author, person_info, product_name, product_url, product_content,
                         product_star,
                         product_meke_all, product_make, product_save, product_time])

            # replys=""
            #评论信息
            # for j in range(len(detil_html.xpath('//div[contains(@class,"page-bottom-outer")]/div[@class="page-container"]//li'))):
            #     reply_name=detil_html.xpath('//div[contains(@class,"page-bottom-outer")]/div[@class="page-container"]//li[{}]//div[contains(@class,"info")]/a/text()'.format(j+1))[0]
            #     product_reply=detil_html.xpath('//div[contains(@class,"page-bottom-outer")]/div[@class="page-container"]//li[{}]//div[@class="right-bottom"]/text()'.format(j+1))[0]
            #     # replys+="用户名"+reply_name+"评论内容"+product_reply+"{{{{{}}}}}"
            #     reply_time1=detil_html.xpath('//div[contains(@class,"page-bottom-outer")]/div[@class="page-container"]//li[{}]//div[@role="question"]//div[contains(@class,"info")]/span/text()'.format(j+1))
            #     reply_time="".join(reply_time1)
            #     reply_url="http://www.xiachufang.com"+detil_html.xpath('//div[contains(@class,"page-bottom-outer")]/div[@class="page-container"]//li[{}]//div[contains(@class,"info")]/a/@href'.format(j+1))[0]
            #     person_reply_html = down_load2(reply_url)
            #     person_reply_info1 = person_reply_html.xpath('//div[@class="gray-font"]/div[1]')[0]
            #     person_reply_info = person_reply_info1.xpath('string(.)').strip().replace(" ","").replace("\n","")
            #     print(person_reply_info)
            #     print(key_word,product_author,person_info,product_name,product_url,product_content,product_star,product_meke_all,product_make,product_save,product_time,reply_name,person_reply_info,reply_time,product_reply)
            #     product_lists.append([key_word,product_author,person_info,product_name,product_url,product_content,product_star,product_meke_all,product_make,product_save,product_time,reply_name,person_reply_info,reply_time,product_reply])
    except:
        with open("下厨房61.csv", "w", encoding="utf-8-sig", newline="") as f:
            k = csv.writer(f, dialect="excel")
            k.writerow(["关键词", "作者", "作者信息", "产品名称", "产品链接","内容", "产品评分", "做过人数", "7天做过","产品收藏", "创建时间"])

            for list in product_lists:
                k.writerow(list)
        pass

print(product_lists)
print(len(product_lists))
with open("下厨房62.csv", "w", encoding="utf-8-sig", newline="") as f:
    k = csv.writer(f, dialect="excel")
    k.writerow(["关键词", "作者", "作者信息", "产品名称", "产品链接","内容", "产品评分", "做过人数", "7天做过","产品收藏", "创建时间"])
    # k.writerow(["关键词", "作者", "作者信息", "产品名称", "产品链接","内容", "产品评分", "做过人数", "7天做过","产品收藏", "创建时间",  "评论人","评论者信息", "评论时间","评论"])

    for list in product_lists:
        k.writerow(list)
Python爬取网站下厨房早餐数据，可另行修改爬取相关数据
12-17
Python爬取网站下厨房早餐数据，可另行修改爬取相关数据,使用pycharm导入，下载相关beautifulsoup4(pip install beautifulsoup4)
【Python xpath爬虫实战】抓取下厨房的菜谱信息，并存储到MySQL数据库
最新发布
qiao1264282918的博客
09-04 1239
获取标题，详情页链接，原材料列表，七天内做过人数，作者字段。获取全部的本周最受欢迎菜谱信息。数据需要存入mysql。
python（爬虫实战）xpath解析方式下厨房
wuyounan的博客
01-18 234
导入模块 请求网页 25行代码轻松拿下
下厨房网站月度最佳栏目菜谱数据获取及分析PLus
一名想当程序员的人
11-20 1426
本文的主题是获取下厨房网站月度最佳栏目近十年数据，最终进行数据清洗、处理后生成所需的数据库表，最终进行数据可视化。用到的技术栈有Python网络爬虫、数据分析、Scala引擎、Flask框架等，其中会重点讲解使用Scala数据处理的过程
下厨房
姚军
11-28 242
下厨房 简洁版 集合的插入和个数统计 #include<cstdio> #include<iostream> #include<string> #include<set> using namespace std; int main(){ string s; set<string> bag; while(cin>>s){...
下厨房网站数据抓取教程：Python爬虫实战
该案例特别适用于初学者或者对网络爬虫技术感兴趣的开发者，目的是为了获取下厨房网站上的菜谱信息。" 知识点详细说明： 1. Python编程基础 - Python是目前广泛使用的一种高级编程语言，它以简洁明了著称，特别...
使用Python爬虫技术抓取下厨房早餐数据
标题中的“Python爬取网站下厨房早餐数据”暗示了一个通过Python编程语言来实现网络爬虫的过程，这主要涉及到网络数据的抓取、解析和数据提取。网络爬虫是一种自动获取网页内容的程序或脚本，它可以按照一定的规则，...
菜谱爬虫,爬取美食中国,豆果,美食天下,下厨房等网站的菜谱.zip
03-06
爬虫通常由搜索引擎、数据挖掘工具、监测系统等应用于网络数据抓取的场景。 爬虫的工作流程包括以下几个关键步骤： URL收集： 爬虫从一个或多个初始URL开始，递归或迭代地发现新的URL，构建一个URL队列。这些URL...
xiaChuFang-python爬虫案例-下厨房网站信息爬取.rar
05-31
本案例以“下厨房”网站为例，深入讲解如何利用Python进行网页数据抓取，旨在帮助读者掌握基本的网络爬虫技能。 首先，我们要了解Python爬虫的基本原理。Python爬虫主要是通过模拟浏览器发送HTTP请求（GET或POST）...
【爬虫练手】下厨房
wyh33200的博客
03-08 179
import requests from bs4 import BeautifulSoup from openpyxl import Workbook wb = Workbook() ws = wb.active list_g = ['菜名','食材','步骤','详细步骤'] ws.append(list_g) headers= { 'User-Agent': 'Mozilla/5....
下厨房-牛客网
Moliay的博客
11-18 252
做一种料理可能需要多种不同的材料,例如做一道辣子鸡丁需要的材料有以下的几种：chicken（鸡肉），chili(辣椒)，pepper（花椒）……..等材料 现在想要做一桌满汉全席，可能用到以下的材料，请问完成一桌 满汉全席需要的材料有多少种。 输入描述 每个输入包含1个测试用例。每个测试用例的第 i 行，表示完成第 i 道菜需要的材料，各种菜需要的材料用空格隔开，输入只包含小...
python爬虫实训_python网络爬虫实践一
weixin_39637614的博客
11-24 339
1、网络蜘蛛 webSpider，第一步查从百度首页爬取源码2、查看自己浏览器的cookieBAIDU_SSP_lcr=https://www.baidu.com/link?url=WHikIXPAqSR3zgfbyzzki6u1Q4biCKXH4N9hJhskEC9-U_kOcWHBDUxFdIdUeSfoEc_9K_l1KSWfDM1m18druQOg6jMHECldh6T0xCl9pFK&w...
python网络爬虫实践一
weixin_34059951的博客
01-02 294
1、网络蜘蛛 webSpider， 第一步查从百度首页爬取源码 2、查看自己浏览器的cookie BAIDU_SSP_lcr=https://www.baidu.com/link?url=WHikIXPAqSR3zgfbyzzki6u1Q4biCKXH4N9hJhskEC9-U_kOcWHBDUxFdIdUeSfoEc_9K_l1KSWfDM1m18druQOg6jMHECldh6T0x...
python 登录新浪微博爬取粉丝信息
在梦中享一刻静谧，等待梦中的你
09-07 9264
最近有个小需求，爬取新浪微博的粉丝信息，弄了好几天，终于搞定，送上代码： 环境： 系统：windows 7 版本：python 3.3 IDE: PyCharm 4.0.4 参考：http://blog.csdn.net/crystal_zero/article/details/51154632 #!/usr/bin/env python3 # -*- coding: utf-8 -*
爬虫实现：根据IP地址反查域名
weixin_34247155的博客
04-13 3053
域名解析与IP地址 域名解析是把域名指向网站空间IP，让人们通过注册的域名可以方便地访问到网站的一种服务；IP地址是网络上标识站点的数字地址，为了方便记忆，采用域名来代替IP地址标识站点地址。域名解析就是域名到IP地址的转换过程，该过程由DNS服务器完成（来自百度百科） 先来了解两个知识点 1、一个域名同一时刻只能对应一个IP地址 2、一个IP地址可以解析绑定多个域名，没有限制 基于以上知识点，假...
requests库爬取汽车之家(get请求)并存入csv文件
周攀的博客
11-24 1446
request库爬取汽车之家（get请求） 首先分析网页结构 查看网页源代码发现标题，图片url，福利和购买跳转链接id都在源码里有： 所以对于标题，福利，和图片url都在标签里，我们可以用xpath进行提取，而购买跳转链接id在json形式的字符串里，所以我们用正则表达式提取 代码如下： title=etree.HTML(req.text).xpath('//*[@id="list"]/ul/li[1]/a/div[4]/text()') newtitle=title[0].replace('\r\
python爬取知乎发现文件存储[txt，json，csv，mongodb]
人生苦短， 还不用Python？
08-15 421
使用XPath import requests import json from lxml import etree from urllib import parse ''' 遇到不懂的问题？Python学习交流群：821460695满足你的需求，资料都已经上传群文件，可以自行下载！ ''' url = 'https://www.zhihu.com/explore' headers = { ...
多线程与协程爬取使用方法，多线程threading，协程gevent和multiprocessing多进程，multiprocessing.dummy多线程
weixin_42357472的博客
12-17 765
#多线程与协程的使用会大大加速抓取速度，多线程算并发会产生重复和写入冲突等问题，协程会自动切换这块比较好点 分别主要是两个模块，多线程threading，协程gevent 1、多线程threading版 ***多线程使用最主要就是给个线程分配内容的问题，这有些小技巧，主要就是最后几句，取模运算分配 import re from lxml import etree import requests i......
Python爬虫利器--异步async框架
weixin_44272011的博客
01-11 1148
这里写目录标题请求逻辑装饰器使用方法定义指定并发量的协程获得协程返回的结果给task 添加回调函数task运行结果 分享一个爬虫，其实只用修改一下爬虫的请求逻辑就会很通用了。在工作中，尽管python慢，但是异步加载的协程可不慢，在请求接口的时候快的飞起。 请求逻辑 这是python的装饰器，其实就是帮助我们打印一下运行日志，挺好用的。 def logged(func): @wraps(func) def wrapper(*args, **kwargs): logger =
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包

打赏作者

loong_XL

你的鼓励将是我创作的最大动力

¥1 ¥2 ¥4 ¥6 ¥10 ¥20
扫码支付：¥1
获取中
扫码支付

您的余额不足，请更换扫码支付或充值

打赏作者

实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值