深圳2024年链家二手房数据集和源码

import csv from bs4 import BeautifulSoup import requests href_list = [] for i in range(1,101): url = f'https://sz.lianjia.com/ershoufang/pg{i}rs%E6%B7%B1%E5%9C%B3/' headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" } data = requests.get(url=url, headers=headers).text soup = BeautifulSoup(data,'lxml') a_tags = soup.find_all('a', class_='noresultRecommend') for a_tag in a_tags: href = a_tag.get('href') href_list.append(href) with open('深圳2024年链家二手房信息_1.csv', mode='w', newline='', encoding='utf-8') as csv_file: fieldnames = [ 'title', 'area', 'community', 'position', 'total_price', 'unit_price', 'hourseType', 'hourseSize', 'direction', 'fitment' ] writer = csv.DictWriter(csv_file, fieldnames=fieldnames) writer.writeheader() for j in range(len(href_list)): try: print(href_list[j]) headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" } data = requests.get(url=href_list[j], headers=headers).text soup = BeautifulSoup(data,'lxml') #title title_h1 = soup.find('h1',class_='main') for titles in title_h1: title = titles #area area_span = soup.find('span',class_='info') area_a = area_span.find('a').text for position_a in area_span.find_all('a')[-1]: position = position_a #community community = soup.find('a',class_='info').text #total_price try: total_price_div = soup.find('div',class_='price') for total_price_span in total_price_div.find('span',class_='total'): total_price = total_price_span except: total_price = None # unit_price try: unit_price_span = soup.find('span',class_='unitPriceValue') unit_price = unit_price_span.get_text(strip=True, separator=" ")[:-4] except: unit_price = None #hourseType hourseType = soup.find('div',class_='mainInfo').text #hourseSize hourseSize_div = soup.find('div',class_='area') for hourseSize_divs in hourseSize_div.find('div',class_='mainInfo'): hourseSize = hourseSize_divs #direction direction_div = soup.find('div',class_='type') for direction_divs in direction_div.find('div',class_='mainInfo'): direction = direction_divs #fitment fitment_div = soup.find('div',class_='type') for fitment_divs in fitment_div.find('div',class_='subInfo'): fitment = fitment_divs[-2:] row_data = { 'title':title, 'area':area_a, 'community':community, 'position':position, 'total_price':total_price, 'unit_price':unit_price, 'hourseType':hourseType, 'hourseSize':hourseSize, 'direction':direction, 'fitment':fitment } writer.writerow(row_data) except: pass