2 Commits

Author SHA1 Message Date
0d68111bc2 Add demo scripts: egress IP check and bulk proxy creation
docs/demo/show-egress-ips.sh probes each healthy proxy from the
discovery API against an IP-echo site; create-kubernetes-proxies.sh and
create-gcp-proxies.sh bulk-create demo Proxies, the gcp one spreading
them across randomly picked EU zones.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-11 22:40:24 +02:00
e1abac3e8f Add plan: demo script for egress IP check via proxies
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-11 22:21:28 +02:00
5 changed files with 287 additions and 0 deletions

94
docs/demo/create-gcp-proxies.sh Executable file
View File

@@ -0,0 +1,94 @@
#!/bin/bash
# Demo: create N Managed proxies backed by the gcp provider, named
# proxy-gcp-demo-1 .. proxy-gcp-demo-N, each in a randomly picked EU zone
# so the fleet gets egress IPs from different locations.
#
# Usage:
# ./create-gcp-proxies.sh <count>
#
# Optional environment:
# NAMESPACE namespace to create the proxies in (default: current context)
# GCP_PROVIDER provider NAME from providers.yaml (default: gcp-eu)
# ZONES space-separated zone list to pick from (default: EU zones below)
set -euo pipefail
if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
echo "usage: $(basename "$0") <count> (positive integer)" >&2
exit 1
fi
count="$1"
provider="${GCP_PROVIDER:-gcp-eu}"
# GCP zones in the EU where e2-micro is generally available. Override with
# ZONES="zone1 zone2 ..." if your project has quota only in some of them.
default_zones=(
europe-west1-b europe-west1-c europe-west1-d # Belgium
europe-west2-a europe-west2-b europe-west2-c # London
europe-west3-a europe-west3-b europe-west3-c # Frankfurt
europe-west4-a europe-west4-b europe-west4-c # Netherlands
europe-west6-a europe-west6-b europe-west6-c # Zurich
europe-west8-a europe-west8-b europe-west8-c # Milan
europe-west9-a europe-west9-b europe-west9-c # Paris
europe-central2-a europe-central2-b europe-central2-c # Warsaw
europe-north1-a europe-north1-b europe-north1-c # Finland
europe-southwest1-a europe-southwest1-b europe-southwest1-c # Madrid
)
if [ -n "${ZONES:-}" ]; then
read -r -a zones <<< "${ZONES}"
else
zones=("${default_zones[@]}")
fi
if ! command -v kubectl &> /dev/null; then
echo "ERROR: kubectl is required but not installed" >&2
exit 1
fi
ns_args=()
if [ -n "${NAMESPACE:-}" ]; then
ns_args=(-n "${NAMESPACE}")
fi
for i in $(seq 1 "${count}"); do
zone="${zones[RANDOM % ${#zones[@]}]}"
echo "Creating proxy-gcp-demo-${i} in ${zone} ..."
kubectl apply "${ns_args[@]}" -f - << EOF
apiVersion: crawl.example.com/v1alpha1
kind: Proxy
metadata:
name: proxy-gcp-demo-${i}
spec:
mode: Managed
provider: ${provider} # must match a provider NAME in providers.yaml
placement:
zone: ${zone}
machineType: e2-micro
# debian-cloud images have no cloud-init, so spec.cloudInit (passed as
# user-data metadata) would be silently ignored there. Ubuntu images do.
image: projects/ubuntu-os-cloud/global/images/family/ubuntu-2404-lts-amd64
port: 3128
cloudInit:
inline: |
#cloud-config
package_update: true
packages:
- squid
write_files:
- path: /etc/squid/conf.d/proxy-operator.conf
content: |
http_access allow all
via off
forwarded_for off
runcmd:
- systemctl restart squid
attributes:
geo: eu
zone: ${zone}
purpose: crawl
EOF
done
echo ""
echo "Created ${count} proxies. VMs take a few minutes to provision and pass"
echo "the health check. Watch them come up with:"
echo " kubectl get px ${ns_args[*]:-} -w"

View File

@@ -0,0 +1,48 @@
#!/bin/bash
# Demo: create N Managed proxies backed by the kubernetes-pod provider,
# named proxy-kubernetes-demo-1 .. proxy-kubernetes-demo-N. Pods share the
# cluster's egress IP — this exercises the full lifecycle, not distinct
# egress paths (use create-gcp-proxies.sh for that).
#
# Usage:
# ./create-kubernetes-proxies.sh <count>
#
# Optional environment:
# NAMESPACE namespace to create the proxies in (default: current context)
set -euo pipefail
if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
echo "usage: $(basename "$0") <count> (positive integer)" >&2
exit 1
fi
count="$1"
if ! command -v kubectl &> /dev/null; then
echo "ERROR: kubectl is required but not installed" >&2
exit 1
fi
ns_args=()
if [ -n "${NAMESPACE:-}" ]; then
ns_args=(-n "${NAMESPACE}")
fi
for i in $(seq 1 "${count}"); do
echo "Creating proxy-kubernetes-demo-${i} ..."
kubectl apply "${ns_args[@]}" -f - << EOF
apiVersion: crawl.example.com/v1alpha1
kind: Proxy
metadata:
name: proxy-kubernetes-demo-${i}
spec:
mode: Managed
provider: kubernetes
attributes:
geo: local
purpose: crawl
EOF
done
echo ""
echo "Created ${count} proxies. Watch them come up with:"
echo " kubectl get px ${ns_args[*]:-} -w"

74
docs/demo/show-egress-ips.sh Executable file
View File

@@ -0,0 +1,74 @@
#!/bin/bash
# Demo: list proxies from the discovery API and show the egress IP each
# healthy one provides, by calling an IP-echo site through it.
#
# Usage:
# ./show-egress-ips.sh <BASE_URL> e.g. ./show-egress-ips.sh localhost:8090
#
# Optional environment:
# TOKEN bearer token for the discovery API (see docs/api.md)
# IP_ECHO_URL site that returns the caller's IP as JSON
# (default: https://api.ipify.org?format=json)
set -euo pipefail
if [ $# -ne 1 ]; then
echo "usage: $(basename "$0") <BASE_URL> (e.g. localhost:8090)" >&2
exit 1
fi
BASE_URL="$1"
IP_ECHO_URL="${IP_ECHO_URL:-https://api.ipify.org?format=json}"
for tool in curl jq; do
if ! command -v "${tool}" &> /dev/null; then
echo "ERROR: ${tool} is required but not installed" >&2
exit 1
fi
done
auth_args=()
if [ -n "${TOKEN:-}" ]; then
auth_args=(-H "Authorization: Bearer ${TOKEN}")
fi
echo "Fetching proxies from ${BASE_URL}/v1/proxies ..."
if ! proxies_json=$(curl -sS --fail "${auth_args[@]}" "${BASE_URL}/v1/proxies"); then
echo "ERROR: could not fetch proxy list from ${BASE_URL}" >&2
exit 1
fi
if ! echo "${proxies_json}" | jq -e . > /dev/null; then
echo "ERROR: response from ${BASE_URL}/v1/proxies is not valid JSON" >&2
exit 1
fi
total=$(echo "${proxies_json}" | jq -r '.count')
echo "Found ${total} proxies"
echo ""
probed=0
skipped=0
failed=0
while IFS= read -r proxy; do
id=$(echo "${proxy}" | jq -r '.id')
ip=$(echo "${proxy}" | jq -r '.ip')
port=$(echo "${proxy}" | jq -r '.port')
healthy=$(echo "${proxy}" | jq -r '.healthy')
if [ "${healthy}" != "true" ]; then
echo "--- skipping ${id} (unhealthy) ---"
echo ""
skipped=$((skipped + 1))
continue
fi
echo "=== via ${id} — http://${ip}:${port} ==="
if response=$(curl -sS --max-time 10 -x "http://${ip}:${port}" "${IP_ECHO_URL}"); then
echo "${response}" | jq . 2> /dev/null || echo "${response}"
probed=$((probed + 1))
else
echo "WARNING: request through ${id} failed" >&2
failed=$((failed + 1))
fi
echo ""
done < <(echo "${proxies_json}" | jq -c '.proxies[]')
echo "Done: ${total} proxies — ${probed} probed, ${skipped} skipped (unhealthy), ${failed} failed"

View File

@@ -0,0 +1,38 @@
# Execution: Demo scripts
Plan: [2026-08-11-2220-demo-scripts.md](../plans/2026-08-11-2220-demo-scripts.md)
- [x] Step 0 — Branch `feat/demo-scripts` + plan commit
- [x] Step 1 — `docs/demo/show-egress-ips.sh`
- [x] Extra (added iteratively, not in the original plan) — proxy-creation scripts
## Step 0 + Step 1
Branched off `main`, committed the plan alone, then wrote
`docs/demo/show-egress-ips.sh`: takes `BASE_URL` as its argument, lists
`/v1/proxies` (bearer auth via optional `TOKEN` env), probes each healthy
proxy with `curl -x http://ip:port` against an IP-echo site
(`IP_ECHO_URL`, default ipify JSON), banners which proxy each request goes
through, skips unhealthy ones, and ends with a probed/skipped/failed
summary. Per user request the script was left uncommitted for iteration
and no verification beyond `bash -n` was run.
## Extra — create-kubernetes-proxies.sh, create-gcp-proxies.sh
Added on the same branch before the first commit:
- `create-kubernetes-proxies.sh <count>` — creates
`proxy-kubernetes-demo-1..N` with the kubernetes provider, spec taken
from `config/samples/proxy_kubernetes.yaml`, applied via
`kubectl apply -f -` heredocs. Optional `NAMESPACE` env.
- `create-gcp-proxies.sh <count>` — creates `proxy-gcp-demo-1..N` from the
user-supplied gcp-eu manifest (e2-micro, Ubuntu 24.04, Squid
cloud-init), each with a zone picked randomly from a hardcoded list of
27 EU zones so the fleet gets egress IPs from different locations.
Env overrides: `ZONES`, `GCP_PROVIDER` (default `gcp-eu`), `NAMESPACE`.
Worth noting: beyond the user's sample manifest, the gcp script also
writes the picked zone into `attributes.zone`, so the discovery API
exposes each proxy's location and leases can select on it. The zone list
is static — if a project lacks quota in some region, `ZONES` narrows the
pool; nothing validates zones against the live project.

View File

@@ -0,0 +1,33 @@
# Plan: Demo script — egress IP check through each healthy proxy
**Created:** 2026-08-11 22:20
## Context
First of a planned series of demo scripts under `docs/demo/`. This one showcases the discovery API end to end without leases: list proxies from `$BASE_URL/v1/proxies`, and for each **healthy** one, call an IP-echo site through it (`curl -x`) to show the egress IP that proxy provides — clearly labeling which proxy each request goes through. The user explicitly asked to start by switching to a new branch. The script will be iterated on: **write it but do not commit it** — the user wants to add things to it before anything is committed.
## Steps
### Step 0 — Branch
Create `feat/demo-scripts` off `main`. Commit only the plan copy (`docs/plans/<timestamp>-demo-scripts.md`, timestamp via `date "+%Y-%m-%d-%H%M"`) per CLAUDE.md — nothing else gets committed this round.
### Step 1 — `docs/demo/show-egress-ips.sh` (new file, executable, left uncommitted)
Style: match [.devcontainer/post-install.sh](.devcontainer/post-install.sh) — `#!/bin/bash`, `set -euo pipefail`, `ERROR:`/`WARNING:` messages, `${VAR}` braces.
Behavior:
1. **Args/env:** `BASE_URL` is `$1` (required; missing → usage text + exit 1, e.g. `usage: show-egress-ips.sh <BASE_URL> (e.g. localhost:8090)`). Optional env: `TOKEN` (bearer token, same name docs/api.md uses; when set, send `Authorization: Bearer $TOKEN`), `IP_ECHO_URL` (default `https://api.ipify.org?format=json` — returns `{"ip":"..."}`).
2. **Dependency check:** `command -v curl`, `command -v jq``ERROR` + exit 1 if missing.
3. **Fetch** `"$BASE_URL/v1/proxies"` once (no server-side `healthy` filter — fetch all so unhealthy ones can be shown as skipped, which makes the demo more informative). Fail with a clear error if curl or JSON parsing fails.
4. **Iterate** proxies with `jq -c '.proxies[]'`; for each, extract `id`, `ip`, `port`, `healthy`:
- unhealthy → print `--- skipping <id> (unhealthy) ---`
- healthy → print a clear banner naming the proxy before the request, e.g. `=== via <id> — http://<ip>:<port> ===`, then `curl -sS --max-time 10 -x "http://${ip}:${port}" "$IP_ECHO_URL"`; print the JSON response. A failed probe prints `WARNING: request through <id> failed` and continues (guard so `set -e` doesn't kill the loop).
5. Finish with a one-line summary: N proxies, M probed, K skipped/failed.
Reference for API shapes: [docs/api.md](docs/api.md) (`proxies[].id/ip/port/healthy`; `curl -x http://ip:port` usage is already documented there and in README).
### Deliberately deferred (user will iterate on the script first)
- No commit of the script, no push beyond the plan commit, no MR, no execution summary, no CHANGELOG — all wait until the user says the script (or script set) is ready.