Rewrite PSN + CityHome scrapers, add price/m² map coloring, ratings system, and status dashboard
- Rewrite PSN scraper to use /api/units-list endpoint (single API call, no HTML parsing) - Fix CityHome scraper: GPS from multiple URL patterns, address from table cells, no 404 retries - Color map markers by price/m² instead of disposition (blue→green→orange→red scale) - Add persistent rating system (favorite/reject) with Flask ratings server and localStorage fallback - Rejected markers show original color at reduced opacity with 🚫 SVG overlay - Favorite markers shown as ⭐ star icons with gold pulse animation - Add "new today" marker logic (scraped_at == today) with larger pulsing green outline - Add filter panel with floor, price, hide-rejected controls and ☰/✕ toggle buttons - Add generate_status.py for scraper run statistics and status.html dashboard - Add scraped_at field to all scrapers for freshness tracking - Update run_all.sh with log capture and status generation Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
116
ratings_server.py
Normal file
116
ratings_server.py
Normal file
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Minimal HTTP API server for persisting apartment ratings.
|
||||
|
||||
GET /api/ratings → returns ratings.json contents
|
||||
POST /api/ratings → saves entire ratings object
|
||||
GET /api/ratings/export → same as GET, but with download header
|
||||
|
||||
Ratings file: /app/data/ratings.json (or ./ratings.json locally)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
PORT = int(os.environ.get("RATINGS_PORT", 8081))
|
||||
DATA_DIR = Path(os.environ.get("DATA_DIR", "."))
|
||||
RATINGS_FILE = DATA_DIR / "ratings.json"
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [ratings] %(levelname)s %(message)s",
|
||||
datefmt="%Y-%m-%dT%H:%M:%S",
|
||||
)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_ratings() -> dict:
|
||||
try:
|
||||
if RATINGS_FILE.exists():
|
||||
return json.loads(RATINGS_FILE.read_text(encoding="utf-8"))
|
||||
except Exception as e:
|
||||
log.error("Failed to load ratings: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
def save_ratings(data: dict) -> None:
|
||||
RATINGS_FILE.write_text(
|
||||
json.dumps(data, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
class RatingsHandler(BaseHTTPRequestHandler):
|
||||
def log_message(self, format, *args):
|
||||
# Suppress default HTTP access log (we use our own)
|
||||
pass
|
||||
|
||||
def _send_json(self, status: int, body: dict, extra_headers=None):
|
||||
payload = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
self.send_header("Access-Control-Allow-Headers", "Content-Type")
|
||||
if extra_headers:
|
||||
for k, v in extra_headers.items():
|
||||
self.send_header(k, v)
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def do_OPTIONS(self):
|
||||
# CORS preflight
|
||||
self.send_response(204)
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
self.send_header("Access-Control-Allow-Headers", "Content-Type")
|
||||
self.end_headers()
|
||||
|
||||
def do_GET(self):
|
||||
if self.path in ("/api/ratings", "/api/ratings/export"):
|
||||
ratings = load_ratings()
|
||||
extra = None
|
||||
if self.path == "/api/ratings/export":
|
||||
extra = {"Content-Disposition": 'attachment; filename="ratings.json"'}
|
||||
log.info("GET %s → %d ratings", self.path, len(ratings))
|
||||
self._send_json(200, ratings, extra)
|
||||
else:
|
||||
self._send_json(404, {"error": "not found"})
|
||||
|
||||
def do_POST(self):
|
||||
if self.path == "/api/ratings":
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
if length == 0:
|
||||
self._send_json(400, {"error": "empty body"})
|
||||
return
|
||||
try:
|
||||
raw = self.rfile.read(length)
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
except Exception as e:
|
||||
log.warning("Bad request body: %s", e)
|
||||
self._send_json(400, {"error": "invalid JSON"})
|
||||
return
|
||||
if not isinstance(data, dict):
|
||||
self._send_json(400, {"error": "expected JSON object"})
|
||||
return
|
||||
save_ratings(data)
|
||||
log.info("POST /api/ratings → saved %d ratings", len(data))
|
||||
self._send_json(200, {"ok": True, "count": len(data)})
|
||||
else:
|
||||
self._send_json(404, {"error": "not found"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
log.info("Ratings server starting on port %d, data dir: %s", PORT, DATA_DIR)
|
||||
log.info("Ratings file: %s", RATINGS_FILE)
|
||||
server = HTTPServer(("0.0.0.0", PORT), RatingsHandler)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
log.info("Stopped.")
|
||||
sys.exit(0)
|
||||
Reference in New Issue
Block a user