Very specific and very dumb data scrapper
Have you ever wanted to collect a complete set of Yu-Gi-Oh! cards, but couldn't find a tool that organized them quite the way you wanted? So, you built a Python web scraper that starts by manually clicking through the cards you want in EdoPro, converts that selection into a readable file, and then automatically downloads all the corresponding PNGs and card info? No, wait... is that really just me? It definitely needs some refinement so I can cut out the manual work entirely, but since it was one of my very first projects, I'm quite fond of it. The code is below, I have no clue if it is usable.
import csv
import json
import os
import requests
from bs4 import BeautifulSoup
# ==========================================
# STEP 1: What website you scrape
# ==========================================
TARGET_URL =
# ==========================================
# STEP 2: List of stuff scraping for
# ==========================================
DATA_FIELDS =
def scrape_website():
headers = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML,"
" like Gecko) Chrome/130.0.0.0 Safari/537.36"
)
}
print(f"Connecting to target website: {TARGET_URL}")
response = requests.get(TARGET_URL, headers=headers)
response.raise_for_status()
soup = BeautifulSoup(response.text, "html.parser")
scraped_data = []
# Find all quote containers on the page
quote_elements = soup.find_all("div", class_="quote")
for element in quote_elements:
quote_text = element.find("span", class_="text").get_text(strip=True)
author_name = element.find("small", class_="author").get_text(strip=True)
tag_elements = element.find_all("a", class_="tag")
tags = [tag.get_text(strip=True) for tag in tag_elements]
scraped_data.append({
"quote": quote_text,
"author": author_name,
"tags": ", ".join(tags),
})
return scraped_data
# ==========================================
# FILE OUTPUT CONFIGURATION
# ==========================================
# Where the scraped files are at:
OUTPUT_DIR = "scraped_output"
def save_files(data):
os.makedirs(OUTPUT_DIR, exist_ok=True)
json_path = os.path.join(OUTPUT_DIR, "scraped_data.json")
csv_path = os.path.join(OUTPUT_DIR, "scraped_data.csv")
# Save as JSON
with open(json_path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=4, ensure_ascii=False)
# Save as CSV
with open(csv_path, "w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=DATA_FIELDS)
writer.writeheader()
writer.writerows(data)
print(f"\nScraping complete! Files are saved at:")
print(f"-> JSON File: {os.path.abspath(json_path)}")
print(f"-> CSV File: {os.path.abspath(csv_path)}")
if __name__ == "__main__":
data = scrape_website()
save_files(data)