NASA天文图像日志数据集

import os import re import csv import time import requests from PIL import Image DEBUG = False BASE_URL = "https://apod.nasa.gov/apod/" HOME_URL = f"{BASE_URL}archivepix.html" DATASET_PATH = os.path.join(".", "dataset") DATASET_IMAGES_PATH = os.path.join(DATASET_PATH, "images") # Creating the dataset folder if it does not exist. if not os.path.exists(DATASET_IMAGES_PATH): os.makedirs(DATASET_IMAGES_PATH, exist_ok=True) SUBPAGE_PATTERN = r"href=\"(ap[^\.]+\.html)\"" IMG_DATE_PATTERN = r"\/ap(\d{2})(\d{2})(\d{2})\.html" IMG_URL_PATTERN = r"<a href=\"(image[^\"]+)\"" TITLE_PATTERN = r"<title>\s*[^\-]+\-\s*([^<]+)" EXPLANATION_PATTERN = r"Explanation:\s*<\/b>([^ъ]+)<b>.+picture:\s*<\/b>" REM_TAGS_PATTERN = r"<[^>]+>" IMG_SIZE = (600, 600) def main(url): html = requests.get(url).text img_urls = re.findall(SUBPAGE_PATTERN, html) metadatum = [] for i, img_url in enumerate(img_urls): subpage_url = f"{BASE_URL}{img_url}" html = requests.get(subpage_url).text # Downloading the image year, month, day = re.findall(IMG_DATE_PATTERN, subpage_url)[0] img_path = os.path.join(DATASET_IMAGES_PATH, f"{year}{month}{day}.jpg") print(f"Image - {img_path}: Downloading...") try: download_img(html, destination=img_path) except Exception: print("An error occurred while downloading the image file!") continue print(f"Image - {img_path}: Downloaded!") # Extracting metadata metadata = extract_metadata(html, image_name=img_path) print(f"Metadata: {metadata}") print() metadatum.append(metadata) if DEBUG: # Save only two images if i == 1: break create_csv( metadatum=metadatum, destination=os.path.join(DATASET_PATH, "metadata.csv") ) def download_img(html, destination): # Getting the image url img_url = re.findall(IMG_URL_PATTERN, html)[0] # Downloading the image img = requests.get(f"{BASE_URL}{img_url}").content # Downloading the image with open(destination, 'wb') as f: f.write(img) # Opening the image im = Image.open(destination) # Verifying if the image is RGB if im.mode != "RGB": # Resizing the downloaded image and saving it im = im.resize(IMG_SIZE) im.save(destination) def extract_metadata(html, image_name): title = re.findall(TITLE_PATTERN, html)[0].strip() explanation = re.findall(EXPLANATION_PATTERN, html)[0] explanation = re.sub(REM_TAGS_PATTERN, "", explanation) explanation = re.sub("\s+", " ", explanation) explanation = explanation.strip() return { "file": image_name.split(os.path.sep)[-1], "title": title, "explanation": explanation, } def create_csv( metadatum, destination, column_names=["file", "title", "explanation"] ): with open(destination, "w", newline="") as f: writer = csv.DictWriter(f, fieldnames=column_names, ) writer.writeheader() writer.writerows(metadatum) if __name__ == "__main__": start_time = time.time() main(HOME_URL) print(f"Elapsed time: {time.time() - start_time:.2f} seconds")