Switch away from combined architecture. Web only.
This commit is contained in:
5
.gitignore
vendored
5
.gitignore
vendored
@@ -1,7 +1,4 @@
|
||||
index.html*
|
||||
/mirrors
|
||||
/node_modules
|
||||
/data
|
||||
node_modules/
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE="/srv/www"
|
||||
URL_LIST="$BASE/mirrors.txt"
|
||||
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "Usage: $0 URL [slug]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
url="$1"
|
||||
if [ $# -ge 2 ]; then
|
||||
slug="$2"
|
||||
else
|
||||
# crude slugify: strip scheme, replace non alnum with underscores
|
||||
slug="$(echo "$url" | sed 's#https\?://##; s#[^a-zA-Z0-9._-]#_#g')"
|
||||
fi
|
||||
|
||||
# Check if URL already exists
|
||||
if grep -q " $url\$" "$URL_LIST" 2>/dev/null; then
|
||||
echo "URL already in list. Not adding again."
|
||||
else
|
||||
echo "$slug $url" >> "$URL_LIST"
|
||||
echo "Added: $slug $url"
|
||||
fi
|
||||
|
||||
# Run update for just this slug
|
||||
"$BASE/update_mirrors.sh" "$slug"
|
||||
@@ -1,61 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import pathlib
|
||||
import html
|
||||
|
||||
BASE = pathlib.Path("/srv/www")
|
||||
URL_LIST = BASE / "mirrors.txt"
|
||||
OUTDIR = BASE / "mirrors"
|
||||
INDEX = BASE / "index.html"
|
||||
|
||||
entries = []
|
||||
|
||||
if URL_LIST.exists():
|
||||
for line in URL_LIST.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
parts = line.split(None, 1)
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
slug, url = parts
|
||||
mirror_dir = OUTDIR / slug
|
||||
if not mirror_dir.exists():
|
||||
# not mirrored yet, but still list it
|
||||
status = " (not downloaded yet)"
|
||||
else:
|
||||
status = ""
|
||||
entries.append((slug, url, status))
|
||||
|
||||
items_html = []
|
||||
for slug, url, status in entries:
|
||||
slug_esc = html.escape(slug)
|
||||
url_esc = html.escape(url)
|
||||
status_esc = html.escape(status)
|
||||
# Link goes to the directory; nginx autoindex or an index file will handle it
|
||||
items_html.append(
|
||||
f'<li><a href="mirrors/{slug_esc}/">{slug_esc}</a>'
|
||||
f' – <code>{url_esc}</code>{status_esc}</li>'
|
||||
)
|
||||
|
||||
html_doc = f"""<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>My Tutorial Mirrors</title>
|
||||
<style>
|
||||
body {{ font-family: sans-serif; max-width: 800px; margin: 2rem auto; }}
|
||||
h1 {{ margin-bottom: 0.5rem; }}
|
||||
code {{ font-size: 0.9em; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Nytegear Mirrors</h1>
|
||||
<p>This page is generated automatically from <code>mirrors.txt</code>.</p>
|
||||
<ul>
|
||||
{''.join(items_html)}
|
||||
</ul>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
INDEX.write_text(html_doc, encoding="utf-8")
|
||||
19
index.html
Normal file
19
index.html
Normal file
@@ -0,0 +1,19 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>My Tutorial Mirrors</title>
|
||||
<style>
|
||||
body { font-family: sans-serif; max-width: 800px; margin: 2rem auto; }
|
||||
h1 { margin-bottom: 0.5rem; }
|
||||
code { font-size: 0.9em; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Nytegear Mirrors</h1>
|
||||
<p>This page is generated automatically from <code>mirrors.txt</code>.</p>
|
||||
<ul>
|
||||
<li><a href="mirrors/wgpu-tutorial/">wgpu-tutorial</a> – <code>https://sotrh.github.io/learn-wgpu/</code></li>
|
||||
</ul>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,190 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import subprocess
|
||||
import datetime as dt
|
||||
from pathlib import Path
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
BASE = Path("/srv/www")
|
||||
DATA_FILE = BASE / "data" / "mirrors.json"
|
||||
MIRROR_ROOT = BASE / "mirrors"
|
||||
LOG_ROOT = BASE / "logs"
|
||||
|
||||
MIRROR_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
LOG_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
DATA_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return dt.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
|
||||
|
||||
|
||||
def load_mirrors() -> list[dict]:
|
||||
with _LOCK:
|
||||
if not DATA_FILE.exists():
|
||||
return []
|
||||
with DATA_FILE.open("r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def save_mirrors(mirrors: list[dict]) -> None:
|
||||
with _LOCK:
|
||||
tmp = DATA_FILE.with_suffix(".tmp")
|
||||
with tmp.open("w", encoding="utf-8") as f:
|
||||
json.dump(mirrors, f, indent=2)
|
||||
tmp.replace(DATA_FILE)
|
||||
|
||||
|
||||
def get_mirror(mirrors: list[dict], slug: str) -> dict | None:
|
||||
for m in mirrors:
|
||||
if m["slug"] == slug:
|
||||
return m
|
||||
return None
|
||||
|
||||
|
||||
def _normalise_categories(raw: str) -> list[str]:
|
||||
# "tutorials, wgpu, rust" -> ["tutorials","wgpu","rust"]
|
||||
parts = [p.strip() for p in raw.split(",")]
|
||||
return [p for p in parts if p]
|
||||
|
||||
|
||||
def add_mirror(slug: str,
|
||||
categories: str,
|
||||
url: str,
|
||||
ignore_robots: bool = False) -> dict:
|
||||
mirrors = load_mirrors()
|
||||
if get_mirror(mirrors, slug) is not None:
|
||||
raise ValueError(f"Mirror with slug '{slug}' already exists")
|
||||
|
||||
cats = _normalise_categories(categories)
|
||||
if not cats:
|
||||
raise ValueError("At least one category is required")
|
||||
|
||||
m = {
|
||||
"slug": slug,
|
||||
"categories": cats,
|
||||
"url": url,
|
||||
"ignore_robots": bool(ignore_robots),
|
||||
"created_at": _now_iso(),
|
||||
"last_updated": None,
|
||||
"status": "queued", # idle | updating | queued | warning | error
|
||||
"last_error": None,
|
||||
}
|
||||
mirrors.append(m)
|
||||
save_mirrors(mirrors)
|
||||
return m
|
||||
|
||||
|
||||
def _set_status(slug: str, *,
|
||||
status: str,
|
||||
last_error: str | None = None,
|
||||
last_updated: str | None = None):
|
||||
mirrors = load_mirrors()
|
||||
m = get_mirror(mirrors, slug)
|
||||
if m is None:
|
||||
return
|
||||
m["status"] = status
|
||||
if last_error is not None:
|
||||
m["last_error"] = last_error
|
||||
if last_updated is not None:
|
||||
m["last_updated"] = last_updated
|
||||
save_mirrors(mirrors)
|
||||
|
||||
|
||||
def update_mirror(slug: str) -> None:
|
||||
"""Run wget mirror for a single slug (blocking in this thread)."""
|
||||
mirrors = load_mirrors()
|
||||
m = get_mirror(mirrors, slug)
|
||||
if m is None:
|
||||
raise ValueError(f"No such mirror: {slug}")
|
||||
|
||||
_set_status(slug, status="updating", last_error=None)
|
||||
|
||||
target_dir = MIRROR_ROOT / slug
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_file = LOG_ROOT / f"{slug}.log"
|
||||
|
||||
robots_setting = "off" if m.get("ignore_robots") else "on"
|
||||
|
||||
cmd = [
|
||||
"wget",
|
||||
"--mirror", # recurse, keep timestamps
|
||||
"--convert-links",
|
||||
"--adjust-extension",
|
||||
"--page-requisites",
|
||||
"--no-parent",
|
||||
"--wait=0.5",
|
||||
"--random-wait",
|
||||
"--limit-rate=50m",
|
||||
"--tries=3",
|
||||
"--retry-connrefused",
|
||||
f"--execute=robots={robots_setting}",
|
||||
"-P",
|
||||
str(target_dir),
|
||||
m["url"],
|
||||
]
|
||||
|
||||
try:
|
||||
with log_file.open("a", encoding="utf-8") as lf:
|
||||
lf.write(f"\n=== {_now_iso()} : Starting mirror of {
|
||||
m['url']} ===\n")
|
||||
lf.flush()
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
stdout=lf,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
lf.write(f"=== {_now_iso()} : wget exited with code {
|
||||
proc.returncode} ===\n")
|
||||
lf.flush()
|
||||
|
||||
# Classify result
|
||||
if proc.returncode == 0:
|
||||
_set_status(slug, status="idle",
|
||||
last_updated=_now_iso(), last_error=None)
|
||||
else:
|
||||
# If we see FINISHED in the log and the directory has content,
|
||||
# treat this as a partial/ok-with-warnings case.
|
||||
text = log_file.read_text(encoding="utf-8", errors="ignore")
|
||||
has_finished = "FINISHED --" in text
|
||||
has_files = any(target_dir.rglob("*"))
|
||||
if has_finished and has_files:
|
||||
_set_status(
|
||||
slug,
|
||||
status="warning",
|
||||
last_updated=_now_iso(),
|
||||
last_error=f"wget exited with {
|
||||
proc.returncode} (partial; see log)",
|
||||
)
|
||||
else:
|
||||
_set_status(
|
||||
slug,
|
||||
status="error",
|
||||
last_error=f"wget exited with {proc.returncode}",
|
||||
)
|
||||
except Exception as e:
|
||||
_set_status(
|
||||
slug,
|
||||
status="error",
|
||||
last_error=f"{type(e).__name__}: {e}",
|
||||
)
|
||||
|
||||
|
||||
def update_all_mirrors(max_workers: int = 3) -> None:
|
||||
mirrors = load_mirrors()
|
||||
slugs = [m["slug"] for m in mirrors]
|
||||
if not slugs:
|
||||
return
|
||||
# Run several in parallel
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
futures = {pool.submit(update_mirror, slug): slug for slug in slugs}
|
||||
for fut in as_completed(futures):
|
||||
slug = futures[fut]
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as e:
|
||||
_set_status(slug, status="error", last_error=f"{
|
||||
type(e).__name__}: {e}")
|
||||
1067
package-lock.json
generated
1067
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,9 +0,0 @@
|
||||
[Unit]
|
||||
Description=Update Offline Website Mirrors
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=aargonian
|
||||
Group=aargonian
|
||||
WorkingDirectory=/srv/www
|
||||
ExecStart=/usr/bin/python3 /srv/www/update_mirrors.py
|
||||
@@ -1,10 +0,0 @@
|
||||
[Unit]
|
||||
Description=Daily update of offline mirrors
|
||||
|
||||
[Timer]
|
||||
OnCalendar=03:00
|
||||
Persistent=true
|
||||
Unit=update-mirrors.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from mirror_manager import update_all_mirrors, update_mirror
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) == 2:
|
||||
slug = sys.argv[1]
|
||||
update_mirror(slug)
|
||||
else:
|
||||
# bump max_workers if you're feeling brave / bandwidth-rich
|
||||
update_all_mirrors(max_workers=8)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user