From 06db0c32834dee729a01bbcceba62a3c969dd5d0 Mon Sep 17 00:00:00 2001 From: Aaron Gorodetzky Date: Fri, 5 Dec 2025 09:31:22 -0500 Subject: [PATCH] Various fixes. Split template files. Add systemd service. --- app.py | 583 ++++-------------- ...ror-manager.service => mirage-web.service} | 9 +- templates/index.html | 287 +++++++++ templates/log.html | 36 ++ 4 files changed, 436 insertions(+), 479 deletions(-) rename systemd/{mirror-manager.service => mirage-web.service} (53%) create mode 100755 templates/index.html create mode 100755 templates/log.html diff --git a/app.py b/app.py index f13fc37..a9d7f78 100755 --- a/app.py +++ b/app.py @@ -1,16 +1,11 @@ #!/usr/bin/env python3 -from mirror_manager import ( - load_mirrors, - add_mirror, - update_mirror, - MIRROR_ROOT, - LOG_ROOT, -) -import re -import html -import subprocess -import threading +from __future__ import annotations + +import os from pathlib import Path +from urllib.parse import urljoin + +import requests from flask import ( Flask, request, @@ -18,356 +13,39 @@ from flask import ( url_for, jsonify, send_from_directory, - render_template_string + render_template, ) - BASE = Path("/srv/www") STATIC_DIR = BASE / "static" STATIC_DIR.mkdir(exist_ok=True) +# Where the Mirage API lives +MIRAGE_API_BASE = os.environ.get( + "MIRAGE_API_BASE", "http://127.0.0.1:5151/api/v1" +) +# How mirrors are exposed over HTTP (for links) +MIRROR_HTTP_BASE = os.environ.get("MIRROR_HTTP_BASE", "/mirrors/") + app = Flask(__name__) -def _run_update_in_background(slug: str): - th = threading.Thread(target=update_mirror, args=(slug,), daemon=True) - th.start() +def _api_url(path: str) -> str: + return urljoin(MIRAGE_API_BASE.rstrip("/") + "/", path.lstrip("/")) -# -------------------- TEMPLATES -------------------- -INDEX_TEMPLATE = r""" - - - - - Mirror Manager - - - -
-
-
-
-

Mirror Manager

-

Local offline mirrors of external sites, grouped by category.

-
-
- - - Running locally - -
-
-
- -
- -
-
-
- Categories: - - {% for cat in categories %} - - {% endfor %} -
-
- -
-
- -
- - - - - - - - - - - - - {% for m in mirrors %} - - - - - - - - - {% endfor %} - {% if mirrors|length == 0 %} - - - - {% endif %} - -
SlugCategoriesURLLast updatedStatus
-
- - {{ m.slug }} - - - View live log - -
- 		-
- {% for c in m.categories %} - {{ c }} - {% endfor %} -
- 		- {{ m.url }} - - {% if m.last_updated %} - {{ m.last_updated }} - {% else %} - never - {% endif %} - - {% set st = m.status or 'idle' %} -
- - {{ st }} -
- 		-
- -
-
- No mirrors yet. Add one below. -
-
-
- - -
-

Add mirror

-
-
- - -
-
- - -
-
- - -
-
- - -
- {% if error %} -

{{ error }}

- {% endif %} - -

- New mirrors are cloned in the background. Status will show as updating until done. -

-
-
- - -
-

Content search

-
- - -
-
-
-
-
- - - - -""" +def api_get(path: str, **kwargs): + resp = requests.get(_api_url(path), timeout=5, **kwargs) + resp.raise_for_status() + return resp.json() -LOG_TEMPLATE = r""" - - - - - Log: {{ slug }} - - - -
-
-
-

Log for {{ slug }}

-

Live tail of wget output (auto-refreshing).

-
- Open mirror -
-
- 

-    

-  

-  
-
-
-"""
+def api_post(path: str, json=None, **kwargs):
+    resp = requests.post(_api_url(path), json=json, timeout=10, **kwargs)
+    return resp
+
 
 # -------------------- ROUTES --------------------
-
-
 @app.route("/static/")
 def static_file(filename):
     return send_from_directory(STATIC_DIR, filename)
@@ -375,7 +53,10 @@ def static_file(filename):
 
 @app.route("/", methods=["GET"])
 def index():
-    mirrors = load_mirrors()
+    # Ask Mirage for mirrors
+    data = api_get("mirrors")
+    mirrors = data.get("mirrors", [])
+
     cats = set()
     rows = []
     for m in mirrors:
@@ -393,25 +74,34 @@ def index():
             "last_updated_raw": raw,
             "last_updated": disp,
         })
-    return render_template_string(INDEX_TEMPLATE, mirrors=rows, categories=sorted(cats), error=None)
+
+    return render_template(
+        "index.html",
+        mirrors=rows,
+        categories=sorted(cats),
+        error=None,
+    )
 
 
 @app.route("/add", methods=["POST"])
 def add_mirror_route():
     slug = (request.form.get("slug") or "").strip()
-    categories = (request.form.get("categories") or "").strip()
+    categories_raw = (request.form.get("categories") or "").strip()
     url = (request.form.get("url") or "").strip()
     ignore_robots = bool(request.form.get("ignore_robots"))
 
     error = None
-    if not slug or not categories or not url:
+    if not slug or not categories_raw or not url:
         error = "Slug, categories, and URL are required."
     elif " " in slug:
         error = "Slug cannot contain spaces."
 
+    categories = [c.strip() for c in categories_raw.split(",") if c.strip()]
+
     if error:
-        # re-render with error
-        mirrors = load_mirrors()
+        # Same re-render pattern as before, but using Mirage API now
+        data = api_get("mirrors")
+        mirrors = data.get("mirrors", [])
         cats = set()
         rows = []
         for m in mirrors:
@@ -429,12 +119,31 @@ def add_mirror_route():
                 "last_updated_raw": raw,
                 "last_updated": disp,
             })
-        return render_template_string(INDEX_TEMPLATE, mirrors=rows, categories=sorted(cats), error=error), 400
+        return (
+            render_template("index.html",
+                            mirrors=rows,
+                            categories=sorted(cats),
+                            error=error),
+            400,
+        )
 
-    try:
-        add_mirror(slug, categories, url, ignore_robots=ignore_robots)
-    except Exception as e:
-        mirrors = load_mirrors()
+    payload = {
+        "slug": slug,
+        "url": url,
+        "categories": categories,
+        "ignore_robots": ignore_robots,
+        "enqueue": True,
+    }
+
+    resp = api_post("mirrors", json=payload)
+    if resp.status_code >= 400:
+        try:
+            msg = resp.json().get("error", resp.text)
+        except Exception:
+            msg = resp.text or "Failed to create mirror."
+        # Same error re-render as above
+        data = api_get("mirrors")
+        mirrors = data.get("mirrors", [])
         cats = set()
         rows = []
         for m in mirrors:
@@ -452,21 +161,27 @@ def add_mirror_route():
                 "last_updated_raw": raw,
                 "last_updated": disp,
             })
-        return render_template_string(INDEX_TEMPLATE, mirrors=rows, categories=sorted(cats), error=str(e)), 400
+        return (
+            render_template("index.html",
+                            mirrors=rows,
+                            categories=sorted(cats),
+                            error=msg),
+            400,
+        )
 
-    _run_update_in_background(slug)
     return redirect(url_for("index"))
 
 
 @app.route("/update/", methods=["POST"])
 def trigger_update(slug):
-    _run_update_in_background(slug)
+    api_post(f"mirrors/{slug}/update")
     return redirect(url_for("index"))
 
 
 @app.route("/status", methods=["GET"])
 def status():
-    mirrors = load_mirrors()
+    data = api_get("mirrors")
+    mirrors = data.get("mirrors", [])
     out = []
     for m in mirrors:
         raw = m.get("last_updated")
@@ -484,78 +199,23 @@ def status():
 
 @app.route("/logs/")
 def log_view(slug):
-    log_path = LOG_ROOT / f"{slug}.log"
-    if not log_path.exists():
-        log_path.touch()
-    return render_template_string(LOG_TEMPLATE, slug=slug)
+    # Template just polls /logs//tail via JS
+    return render_template("log.html", slug=slug)
 
 
 @app.route("/logs//tail")
 def log_tail(slug):
-    log_path = LOG_ROOT / f"{slug}.log"
-    if not log_path.exists():
-        return "", 200
     try:
-        with log_path.open("rb") as f:
-            f.seek(0, 2)
-            size = f.tell()
-            block = 65536
-            if size <= block:
-                f.seek(0)
-                data = f.read()
-            else:
-                f.seek(-block, 2)
-                data = f.read()
-        return data.decode("utf-8", errors="replace")
-    except OSError:
+        resp = requests.get(_api_url(f"mirrors/{slug}/log"),
+                            params={"bytes": 65536},
+                            timeout=5)
+    except requests.RequestException:
         return "", 200
 
+    if resp.status_code >= 400:
+        return "", 200
 
-def strip_html(text: str) -> str:
-    # Remove script and style blocks first
-    text = re.sub(
-        r")<[^<]*)*",
-        " ",
-        text,
-        flags=re.IGNORECASE,
-    )
-    text = re.sub(
-        r")<[^<]*)*",
-        " ",
-        text,
-        flags=re.IGNORECASE,
-    )
-    # Strip all remaining tags
-    text = re.sub(r"<[^>]+>", " ", text)
-    # Unescape HTML entities (& → &, etc.)
-    text = html.unescape(text)
-    # Collapse whitespace
-    text = re.sub(r"\s+", " ", text).strip()
-    return text
-
-
-def make_snippet(text: str,
-                 query: str,
-                 radius: int = 80,
-                 max_len: int = 240) -> str:
-    if not text:
-        return ""
-    lower = text.lower()
-    qlower = query.lower()
-    idx = lower.find(qlower)
-    if idx == -1:
-        snippet = text[:max_len]
-        if len(text) > max_len:
-            snippet += "…"
-        return snippet
-    start = max(0, idx - radius)
-    end = min(len(text), idx + len(query) + radius)
-    snippet = text[start:end]
-    if start > 0:
-        snippet = "…" + snippet
-    if end < len(text):
-        snippet += "…"
-    return snippet
+    return resp.text, 200, {"Content-Type": "text/plain; charset=utf-8"}
 
 
 @app.route("/search", methods=["GET"])
@@ -565,73 +225,46 @@ def content_search():
         return jsonify({"results": []})
 
     try:
-        proc = subprocess.run(
-            [
-                "rg",
-                "--line-number",
-                "--no-heading",
-                "--color", "never",
-                "--max-count", "5",  # per file
-                "--type-add", "page:*.{html,htm,md,markdown,txt}",
-                "-tpage",
-                q,
-                str(MIRROR_ROOT),
-            ],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.DEVNULL,
-            text=True,
-            timeout=10,
-        )
-    except FileNotFoundError:
+        resp = requests.get(_api_url("search"),
+                            params={"q": q},
+                            timeout=10)
+    except requests.RequestException:
         return jsonify({
             "results": [{
                 "path": "(error)",
                 "line": 0,
                 "url": "",
-                "snippet": "ripgrep (rg) is not installed."
-            }]
-        })
-    except subprocess.TimeoutExpired:
-        return jsonify({
-            "results": [{
-                "path": "(error)",
-                "line": 0,
-                "url": "",
-                "snippet": "rg timed out."
+                "snippet": "Search request to Mirage API failed.",
             }]
         })
 
+    if resp.status_code >= 400:
+        return jsonify({
+            "results": [{
+                "path": "(error)",
+                "line": 0,
+                "url": "",
+                "snippet": f"Mirage API returned {resp.status_code}.",
+            }]
+        })
+
+    data = resp.json()
     results = []
-    for line in proc.stdout.splitlines():
-        parts = line.split(":", 2)
-        if len(parts) != 3:
-            continue
-        path, lineno, raw_content = parts
+    for r in data.get("results", []):
+        rel_path = r.get("path", "")
+        lineno = int(r.get("line", 0))
+        snippet = r.get("snippet") or ""
 
-        # Strip HTML/JS/CSS markup from this line before making a snippet
-        text_content = strip_html(raw_content)
-        if not text_content:
-            continue
-
-        snippet = make_snippet(text_content, q)
-
-        try:
-            rel_path = str(Path(path).relative_to(MIRROR_ROOT))
-        except ValueError:
-            rel_path = path
-
-        url = "/mirrors/" + rel_path.replace("\\", "/")
+        # Web UI decides how mirrors are exposed over HTTP:
+        url = MIRROR_HTTP_BASE.rstrip("/") + "/" + rel_path.replace("\\", "/")
 
         results.append({
             "path": rel_path,
-            "line": int(lineno),
+            "line": lineno,
             "url": url,
             "snippet": snippet,
         })
 
-        if len(results) >= 50:
-            break
-
     return jsonify({"results": results})
 
 
diff --git a/systemd/mirror-manager.service b/systemd/mirage-web.service
similarity index 53%
rename from systemd/mirror-manager.service
rename to systemd/mirage-web.service
index 427e9d6..5af11bc 100644
--- a/systemd/mirror-manager.service
+++ b/systemd/mirage-web.service
@@ -1,13 +1,14 @@
 [Unit]
-Description=Mirror Manager Flask App
+Description=Mirage-Web Interface
+Requires=mirage-api.service
 After=network.target
 
 [Service]
-User=aargonian
-Group=aargonian
+User=mirage
+Group=mirage
 WorkingDirectory=/srv/www
 Environment="FLASK_ENV=production"
-ExecStart=/usr/bin/python3 /srv/www/app.py
+ExecStart=/usr/bin/python3 /srv/www/mirage/mirage-web/app.py
 Restart=on-failure
 RestartSec=5
 
diff --git a/templates/index.html b/templates/index.html
new file mode 100755
index 0000000..fe036fc
--- /dev/null
+++ b/templates/index.html
@@ -0,0 +1,287 @@
+
+
+
+  
+  Mirror Manager
+  
+
+
+  

+    

+      

+        

+          
Mirror Manager

+          
Local offline mirrors of external sites, grouped by category.

+        

+        

+          
+            
+            Running locally
+          
+        

+      

+    

+
+    

+      
+      

+        

+          

+            Categories:
+            
+            {% for cat in categories %}
+            
+            {% endfor %}
+          

+          

+            
+          

+        

+
+        

+          
+            
+              
+                
+                
+                
+                
+                
+                
+              
+            
+            
+              {% for m in mirrors %}
+              
+                
+                
+                
+                
+                
+                
+              
+              {% endfor %}
+              {% if mirrors|length == 0 %}
+              
+                
+              
+              {% endif %}
+            
+          
SlugCategoriesURLLast updatedStatus

+                  

+                    
+                      {{ m.slug }}
+                    
+                    
+                      View live log
+                    
+                  

+                		
+                  

+                    {% for c in m.categories %}
+                    {{ c }}
+                    {% endfor %}
+                  

+                		
+                  {{ m.url }}
+                
+                  {% if m.last_updated %}
+                    {{ m.last_updated }}
+                  {% else %}
+                    never
+                  {% endif %}
+                
+                  {% set st = m.status or 'idle' %}
+                  

+                    
+                    {{ st }}
+                  

+                		
+                  

+                    
+                  

+                

+                  No mirrors yet. Add one below.
+                

+        

+      

+
+      
+      

+        
Add mirror

+        

+          

+            
+            
+          

+          

+            
+            
+          

+          

+            
+            
+          

+          

+            
+            
+          

+          {% if error %}
+          
{{ error }}

+          {% endif %}
+          
+          

+            New mirrors are cloned in the background. Status will show as updating until done.
+          

+        

+      

+
+      
+      

+        
Content search

+        

+          
+          
+        

+        

+      

+    

+  

+
+  
+
+
diff --git a/templates/log.html b/templates/log.html
new file mode 100755
index 0000000..386f053
--- /dev/null
+++ b/templates/log.html
@@ -0,0 +1,36 @@
+
+
+
+  
+  Log: {{ slug }}
+  
+
+
+  

+    

+      

+        
Log for {{ slug }}

+        
Live tail of wget output (auto-refreshing).

+      

+      Open mirror
+    

+    

+      

+    

+  

+  
+
+