From c9f23f83e78154acc470cc5cfed609575a57f00d Mon Sep 17 00:00:00 2001 From: Kasper Plougmann Date: Tue, 8 Jul 2025 12:25:45 +0200 Subject: [PATCH] Create fetch_torrents.py --- fetch_torrents.py | 172 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 fetch_torrents.py diff --git a/fetch_torrents.py b/fetch_torrents.py new file mode 100644 index 0000000..7c2a258 --- /dev/null +++ b/fetch_torrents.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +import os +import re +import requests +import logging +import time +import shutil +from bs4 import BeautifulSoup + +# Configure logging +log_file = "/logs/fetch_torrents.log" +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s: %(message)s", + handlers=[ + logging.StreamHandler(), + logging.FileHandler(log_file) + ] +) + +watch_dir = "/watch" + +def download_torrent(name, url): + dest = os.path.join(watch_dir, f"{name}.torrent") + try: + logging.info(f"Fetching {url} ...") + r = requests.get(url, timeout=30) + r.raise_for_status() + with open(dest, "wb") as f: + f.write(r.content) + logging.info(f"Saved {dest}") + return True + except Exception as e: + logging.error(f"Failed to download {url}: {e}") + return False + +def fetch_ubuntu_lts(): + url = "https://releases.ubuntu.com/" + try: + r = requests.get(url, timeout=30) + r.raise_for_status() + soup = BeautifulSoup(r.text, "html.parser") + + lts_versions = [] + for link in soup.find_all('a', href=True): + href = link['href'] + if re.match(r"^\d{2}\.\d{2}/$", href) and "lts" in link.text.lower(): + lts_versions.append(href.strip("/")) + + if not lts_versions: + logging.warning("No Ubuntu LTS versions found.") + return False + + latest_lts = sorted(lts_versions, reverse=True)[0] + torrent_url = f"https://releases.ubuntu.com/{latest_lts}/ubuntu-{latest_lts}-desktop-amd64.iso.torrent" + return download_torrent(f"ubuntu-{latest_lts}", torrent_url) + except Exception as e: + logging.error(f"Ubuntu fetch error: {e}") + return False + +def fetch_debian_stable(): + url = "https://cdimage.debian.org/debian-cd/current/amd64/iso-dvd/" + try: + r = requests.get(url, timeout=30) + r.raise_for_status() + soup = BeautifulSoup(r.text, "html.parser") + + for link in soup.find_all('a', href=True): + href = link['href'] + if "-DVD-1.iso.torrent" in href: + torrent_url = url + href + name = href.replace(".iso.torrent", "") + return download_torrent(name, torrent_url) + logging.warning("No Debian DVD-1 torrent found.") + return False + except Exception as e: + logging.error(f"Debian fetch error: {e}") + return False + +def fetch_fedora_latest(): + url = "https://getfedora.org/en/workstation/download/" + try: + r = requests.get(url, timeout=30) + r.raise_for_status() + soup = BeautifulSoup(r.text, "html.parser") + + for link in soup.find_all('a', href=True): + href = link['href'] + if href.endswith(".torrent") and "Workstation" in href: + torrent_url = href + name = os.path.basename(href).replace(".torrent", "") + return download_torrent(name, torrent_url) + logging.warning("No Fedora torrent found.") + return False + except Exception as e: + logging.error(f"Fedora fetch error: {e}") + return False + +def fetch_arch_latest(): + torrent_url = "https://geo.mirror.pkgbuild.com/iso/latest/archlinux-x86_64.iso.torrent" + return download_torrent("archlinux-latest", torrent_url) + +def fetch_kali_latest(): + url = "https://www.kali.org/get-kali/#kali-installer-images" + try: + r = requests.get(url, timeout=30) + r.raise_for_status() + soup = BeautifulSoup(r.text, "html.parser") + + for link in soup.find_all('a', href=True): + href = link['href'] + if href.endswith(".torrent"): + torrent_url = href + name = os.path.basename(href).replace(".torrent", "") + return download_torrent(name, torrent_url) + logging.warning("No Kali torrent found.") + return False + except Exception as e: + logging.error(f"Kali fetch error: {e}") + return False + +def fetch_distrowatch_torrents(): + enable = os.getenv("ENABLE_DISTROWATCH", "false").lower() == "true" + if not enable: + logging.info("DistroWatch fetching disabled.") + return False + + filters = os.getenv("DISTROWATCH_FILTER", "").split(",") + filters = [f.strip().lower() for f in filters if f.strip()] + + url = "https://distrowatch.com/dwres.php?resource=torrents" + try: + r = requests.get(url, timeout=30) + r.raise_for_status() + soup = BeautifulSoup(r.text, "html.parser") + + count = 0 + for link in soup.find_all('a', href=True): + href = link['href'] + if href.endswith(".torrent"): + distro_name = link.text.lower() + if filters and not any(f in distro_name for f in filters): + continue + + torrent_url = href if href.startswith("http") else f"https://distrowatch.com/{href}" + name = os.path.basename(torrent_url).replace(".torrent", "") + if download_torrent(name, torrent_url): + count += 1 + logging.info(f"DistroWatch torrents fetched: {count}") + return True + except Exception as e: + logging.error(f"DistroWatch fetch error: {e}") + return False + +if __name__ == "__main__": + start_time = time.time() + logging.info("Starting torrent fetch run.") + + success_count = 0 + failure_count = 0 + + for func in [fetch_ubuntu_lts, fetch_debian_stable, fetch_fedora_latest, fetch_arch_latest, fetch_kali_latest, fetch_distrowatch_torrents]: + if func(): + success_count += 1 + else: + failure_count += 1 + + total, used, free = shutil.disk_usage("/downloads") + logging.info(f"Downloads folder usage: {used // (2**30)} GB used / {total // (2**30)} GB total") + + elapsed = time.time() - start_time + logging.info(f"Run complete in {elapsed:.2f} seconds. {success_count} successful, {failure_count} failed.")