Compare commits
8 Commits
801a9fbe5f
...
0fe62ef314
| Author | SHA1 | Date | |
|---|---|---|---|
| 0fe62ef314 | |||
| d595a93d36 | |||
| c489832ce7 | |||
| add120c033 | |||
| 8176a5eef8 | |||
| 4aa3d47e3c | |||
| f6d50e4744 | |||
| 223b6a8fd6 |
@@ -63,7 +63,15 @@
|
||||
"Bash(grep -n 'func Channel' -A8 __CMDSUB_OUTPUT__/sigs.k8s.io/controller-runtime@v0.24.1/pkg/source/source.go)",
|
||||
"Bash(grep -n 'type GenericEvent' __CMDSUB_OUTPUT__/sigs.k8s.io/controller-runtime@v0.24.1/pkg/event/event.go)",
|
||||
"Bash(KUBEBUILDER_ASSETS=__TRACKED_VAR__/bin/k8s/1.36.2-darwin-arm64 go test -race ./...)",
|
||||
"Bash(cat >> *)"
|
||||
"Bash(cat >> *)",
|
||||
"Bash(make run-dev *)",
|
||||
"Bash(kubectl get *)",
|
||||
"Bash(kubectl delete *)",
|
||||
"Bash(make docker-build *)",
|
||||
"Bash(kind load *)",
|
||||
"Bash(make deploy *)",
|
||||
"Bash(kubectl -n egress-proxies-operator-system rollout status deploy/egress-proxies-operator-controller-manager --timeout=120s)",
|
||||
"Bash(kubectl -n egress-proxies-operator-system rollout restart deploy/egress-proxies-operator-controller-manager)"
|
||||
],
|
||||
"additionalDirectories": [
|
||||
"/Users/jan.novak/srv/go/egress-proxies-operator/.claude",
|
||||
|
||||
24
CHANGELOG.md
24
CHANGELOG.md
@@ -1 +1,25 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-08-10 09:34 CEST — kind e2e verified; fix Squid OOM in containers; quickstart goes in-cluster
|
||||
|
||||
- Full end-to-end pass on a throwaway kind cluster: Squid pod Ready with a real CONNECT
|
||||
probe (89 ms), lease grant/report/cooldown-409/release through the discovery API,
|
||||
finalizer cleanup on delete.
|
||||
- Fixed the kubernetes provider's generated squid.conf: `max_filedescriptors 1024`
|
||||
(squid sizes FD tables from the container's effectively-unlimited RLIMIT_NOFILE and
|
||||
was OOM-killed at startup under kind/containerd) + `cache_mem 16 MB`.
|
||||
- README quickstart now deploys the operator in-cluster: `make run-dev` on a laptop
|
||||
cannot reach kind pod IPs, so health probes fail by construction there (documented).
|
||||
|
||||
## 2026-08-09 17:27 CEST — Operator wired end to end: reconciler, health, leases, discovery, GC, two providers
|
||||
|
||||
- `cmd/main.go` is now the full composition root: `--providers-config` (required, fail-fast),
|
||||
`--discovery-addr`, `--proxy-namespace`, `--health-workers`, `--gc-interval`, `--gc-min-age`,
|
||||
`--gc-allow-namespaced`, `--lease-cooldown`, `--max-lease-ttl`; wires the kubernetes + gcp
|
||||
providers (metrics-instrumented), health engine, lease store, discovery API, orphan GC, and
|
||||
Prometheus metrics onto one manager.
|
||||
- Deploy manifests: providers ConfigMap mount, optional `DISCOVERY_TOKEN` Secret env,
|
||||
discovery port 8090 + Service; pods RBAC for the kubernetes provider.
|
||||
- Samples for all three proxy flavors + providers-config; `make run-dev` for local development.
|
||||
- README rewritten (kind quickstart, GCP setup, the two load-bearing caveats); architecture doc
|
||||
completed with components table and the full decision log.
|
||||
|
||||
6
Makefile
6
Makefile
@@ -61,7 +61,7 @@ vet: ## Run go vet against code.
|
||||
|
||||
.PHONY: test
|
||||
test: manifests generate fmt vet setup-envtest ## Run tests.
|
||||
KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
|
||||
KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test -race $$(go list ./... | grep -v /e2e) -coverprofile cover.out
|
||||
|
||||
# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
|
||||
# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
|
||||
@@ -116,6 +116,10 @@ build: manifests generate fmt vet ## Build manager binary.
|
||||
run: manifests generate fmt vet ## Run a controller from your host.
|
||||
go run ./cmd/main.go
|
||||
|
||||
.PHONY: run-dev
|
||||
run-dev: manifests generate fmt vet ## Run locally against the current kubeconfig with the kubernetes-pod provider.
|
||||
go run ./cmd/main.go --providers-config hack/providers-dev.yaml --metrics-bind-address :8080 --metrics-secure=false
|
||||
|
||||
# If you wish to build the manager image targeting other platforms you can use the --platform flag.
|
||||
# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
|
||||
# More info: https://docs.docker.com/develop/develop-images/build_enhancements/
|
||||
|
||||
241
README.md
241
README.md
@@ -1,135 +1,186 @@
|
||||
# egress-proxies-operator
|
||||
// TODO(user): Add simple overview of use/purpose
|
||||
|
||||
## Description
|
||||
// TODO(user): An in-depth paragraph about your project and overview of use
|
||||
A Kubernetes operator that manages a fleet of HTTP egress proxies for
|
||||
crawling: each proxy is a `Proxy` custom resource that the operator
|
||||
provisions (or merely tracks), actively health-checks **through the proxy
|
||||
itself**, and hands out to crawler clients via an HTTP list/lease API.
|
||||
|
||||
## Getting Started
|
||||
## Architecture in 60 seconds
|
||||
|
||||
### Prerequisites
|
||||
- go version v1.24.6+
|
||||
- docker version 17.03+.
|
||||
- kubectl version v1.11.3+.
|
||||
- Access to a Kubernetes v1.11.3+ cluster.
|
||||
- **`Proxy` CRD** (`crawl.example.com/v1alpha1`, namespaced, `kubectl get px`):
|
||||
`Managed` proxies are provisioned by a configured provider; `External`
|
||||
proxies exist elsewhere and are only tracked and health-checked.
|
||||
- **Reconciler** — a crash-safe state machine: every reconcile derives one
|
||||
action from (spec, status, provider Get). Proxies are **immutable
|
||||
cattle**: any meaningful spec change (placement, cloud-init, port)
|
||||
deletes and recreates the VM — never in-place mutation.
|
||||
- **Providers** behind one minimal interface: `kubernetes` (a real Squid
|
||||
pod in this cluster — local dev/CI) and `gcp` (Compute Engine VMs with
|
||||
ephemeral external IPs — the real egress fleet). Config is a YAML file
|
||||
(`--providers-config`) with named instances (`gcp-eu`, `gcp-us`, ...).
|
||||
- **Health engine** probes every proxy by fetching a URL *through* it (a
|
||||
real CONNECT tunnel — a proxy that accepts TCP but can't egress goes
|
||||
Unhealthy), with threshold logic and transition-only status writes.
|
||||
- **Discovery API** (`:8090`): list healthy proxies filtered by
|
||||
attributes, lease one (least-loaded, TTL-based), release, and report
|
||||
rate-limiting — reports put the proxy in a per-target cooldown.
|
||||
- **Orphan GC** sweeps each provider for tagged instances whose owning CR
|
||||
is gone — the safety net for crashes mid-create.
|
||||
|
||||
### To Deploy on the cluster
|
||||
**Build and push your image to the location specified by `IMG`:**
|
||||
Details, diagrams, and recorded design decisions: [docs/architecture.md](docs/architecture.md).
|
||||
|
||||
## Quickstart on kind (~5 minutes)
|
||||
|
||||
Requires: kind, kubectl, docker, Go 1.26, jq (optional). The
|
||||
kubernetes-pod provider needs no cloud account — proxies are real
|
||||
`ubuntu/squid` pods in the kind cluster itself.
|
||||
|
||||
The operator runs **in-cluster** for this quickstart. (Running it on your
|
||||
laptop with `make run-dev` provisions pods fine, but the health probe then
|
||||
originates on your machine, which cannot reach kind's pod IPs — the proxy
|
||||
would sit at `Unhealthy` forever. In-cluster, probes run where the pod
|
||||
network is routable.)
|
||||
|
||||
```sh
|
||||
make docker-build docker-push IMG=<some-registry>/egress-proxies-operator:tag
|
||||
kind create cluster --name proxy-operator-demo
|
||||
make install # install the CRD
|
||||
make docker-build IMG=egress-proxies-operator:dev
|
||||
kind load docker-image egress-proxies-operator:dev --name proxy-operator-demo
|
||||
make deploy IMG=egress-proxies-operator:dev
|
||||
kubectl -n egress-proxies-operator-system rollout status deploy/egress-proxies-operator-controller-manager
|
||||
```
|
||||
|
||||
**NOTE:** This image ought to be published in the personal registry you specified.
|
||||
And it is required to have access to pull the image from the working environment.
|
||||
Make sure you have the proper permission to the registry if the above commands don’t work.
|
||||
|
||||
**Install the CRDs into the cluster:**
|
||||
Create a proxy and watch it come up:
|
||||
|
||||
```sh
|
||||
make install
|
||||
kubectl apply -f config/samples/proxy_kubernetes.yaml
|
||||
kubectl get px -w
|
||||
# NAME MODE PROVIDER PHASE IP HEALTHY
|
||||
# proxy-kubernetes-sample Managed kubernetes Ready 10.244.x.x True
|
||||
```
|
||||
|
||||
**Deploy the Manager to the cluster with the image specified by `IMG`:**
|
||||
Once it's `Ready`, port-forward the discovery API and use it:
|
||||
|
||||
```sh
|
||||
make deploy IMG=<some-registry>/egress-proxies-operator:tag
|
||||
kubectl -n egress-proxies-operator-system port-forward \
|
||||
svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090 &
|
||||
```
|
||||
|
||||
> **NOTE**: If you encounter RBAC errors, you may need to grant yourself cluster-admin
|
||||
privileges or be logged in as admin.
|
||||
|
||||
**Create instances of your solution**
|
||||
You can apply the samples (examples) from the config/sample:
|
||||
|
||||
```sh
|
||||
kubectl apply -k config/samples/
|
||||
# List healthy proxies
|
||||
curl -s 'localhost:8090/v1/proxies?healthy=true' | jq
|
||||
|
||||
# Lease one (5-minute TTL)
|
||||
curl -s -XPOST localhost:8090/v1/leases \
|
||||
-d '{"selector":{"geo":"local"},"ttlSeconds":300}' | jq
|
||||
# → {"leaseID":"...", "proxy":{"id":"default/proxy-kubernetes-sample", "ip":..., ...}}
|
||||
|
||||
# Actually crawl through it (from inside the cluster, or port-forward the pod)
|
||||
# curl -x http://<proxy-ip>:3128 https://example.com
|
||||
|
||||
# Report the proxy got rate-limited by a site → 15-minute cooldown for that target
|
||||
curl -s -XPOST localhost:8090/v1/leases/<leaseID>/report \
|
||||
-d '{"result":"rate_limited","target":"example.com"}'
|
||||
|
||||
# Release early (idempotent — 204 both times)
|
||||
curl -si -XDELETE localhost:8090/v1/leases/<leaseID>
|
||||
```
|
||||
|
||||
>**NOTE**: Ensure that the samples has default values to test it out.
|
||||
|
||||
### To Uninstall
|
||||
**Delete the instances (CRs) from the cluster:**
|
||||
Tear down:
|
||||
|
||||
```sh
|
||||
kubectl delete -k config/samples/
|
||||
kubectl delete -f config/samples/proxy_kubernetes.yaml # finalizer deletes the pod
|
||||
kind delete cluster --name proxy-operator-demo
|
||||
```
|
||||
|
||||
**Delete the APIs(CRDs) from the cluster:**
|
||||
## Deploying in-cluster
|
||||
|
||||
```sh
|
||||
make uninstall
|
||||
make docker-build IMG=<registry>/egress-proxies-operator:dev
|
||||
make deploy IMG=<registry>/egress-proxies-operator:dev
|
||||
```
|
||||
|
||||
**UnDeploy the controller from the cluster:**
|
||||
- Provider config comes from the `providers-config` ConfigMap
|
||||
([config/manager/providers_config.yaml](config/manager/providers_config.yaml));
|
||||
the default ships only the kubernetes provider.
|
||||
- The discovery API is exposed by the
|
||||
`controller-manager-discovery-service` Service on port 8090.
|
||||
- Auth: create the token Secret, or the API serves **unauthenticated**
|
||||
(it warns loudly at startup):
|
||||
|
||||
```sh
|
||||
make undeploy
|
||||
kubectl -n egress-proxies-operator-system create secret generic discovery-token \
|
||||
--from-literal=token="$(openssl rand -hex 24)"
|
||||
```
|
||||
|
||||
## Project Distribution
|
||||
## GCP setup
|
||||
|
||||
Following the options to release and provide this solution to the users.
|
||||
1. Add a `gcp` entry to the providers config (see
|
||||
[config/samples/providers-config.yaml](config/samples/providers-config.yaml)) —
|
||||
only `project` is required.
|
||||
2. Credentials are **Application Default Credentials**: workload identity
|
||||
in-cluster, `gcloud auth application-default login` locally. No
|
||||
key-file plumbing exists.
|
||||
3. The identity needs `roles/compute.instanceAdmin.v1` on the project —
|
||||
plus `roles/iam.serviceAccountUser` if instances attach a service
|
||||
account.
|
||||
4. Managed GCP proxies must set all of `placement.zone`,
|
||||
`placement.machineType`, and `placement.image`
|
||||
(see [config/samples/proxy_gcp.yaml](config/samples/proxy_gcp.yaml),
|
||||
which also installs Squid via cloud-init). A missing field fails the
|
||||
Proxy with a message naming it.
|
||||
|
||||
### By providing a bundle with all YAML files
|
||||
Cloud-init from a Secret: the Secret **must** carry the label
|
||||
`crawl.example.com/cloud-init: "true"` — the operator's cache only holds
|
||||
labelled Secrets, so an unlabelled one is invisible (the Proxy reports
|
||||
`CloudInitError`). Rotating the Secret's content triggers VM replacement.
|
||||
|
||||
1. Build the installer for the image built and published in the registry:
|
||||
## Caveats — read these two
|
||||
|
||||
**Changing a proxy changes its IP.** Proxies are immutable cattle: editing
|
||||
`placement`, `cloudInit` (or rotating its Secret), or `port` deletes the
|
||||
VM and creates a replacement with the **same name but a new IP**. Clients
|
||||
discover the new address via the discovery API; anything that pinned the
|
||||
old IP breaks by design.
|
||||
|
||||
**Operator restart drops all leases and cooldowns.** Lease state is
|
||||
in-memory (`replicas: 1` accordingly). Clients must tolerate a lease
|
||||
vanishing — requests through the proxy keep working; they just re-lease.
|
||||
The lease store sits behind an interface so a persistent backend can
|
||||
replace it without touching the API handlers.
|
||||
|
||||
Smaller notes:
|
||||
|
||||
- `status.lastHealthCheckTime` is the time of the last *status-affecting*
|
||||
probe, not the most recent probe — status writes are transition-only by
|
||||
design. True probe recency lives in the metrics
|
||||
(`proxy_operator_healthcheck_*`).
|
||||
- The discovery API is served by every replica but is not leader-elected;
|
||||
the operator ships with `replicas: 1` (see the lease caveat above).
|
||||
|
||||
## Version pins
|
||||
|
||||
Built and verified against the spec's pins with **no substitutions
|
||||
needed**: Go 1.26, kubebuilder v4.15.0, controller-runtime v0.24.1,
|
||||
k8s.io/* v0.36.3 (Kubernetes 1.36 API level), controller-tools v0.21.0,
|
||||
cloud.google.com/go/compute v1.65.0. envtest uses the 1.36.2 binary
|
||||
bundle (the latest 1.36 patch with published binaries — do not "fix" the
|
||||
Makefile's derived version to 1.36.3, which has none).
|
||||
|
||||
## Development
|
||||
|
||||
```sh
|
||||
make build-installer IMG=<some-registry>/egress-proxies-operator:tag
|
||||
make test # unit + envtest suites, with -race (sets up envtest binaries itself)
|
||||
go test -short ./... # skip the envtest suite
|
||||
make run-dev # run against the current kubeconfig context
|
||||
```
|
||||
|
||||
**NOTE:** The makefile target mentioned above generates an 'install.yaml'
|
||||
file in the dist directory. This file contains all the resources built
|
||||
with Kustomize, which are necessary to install this project without its
|
||||
dependencies.
|
||||
|
||||
2. Using the installer
|
||||
|
||||
Users can just run 'kubectl apply -f <URL for YAML BUNDLE>' to install
|
||||
the project, i.e.:
|
||||
|
||||
```sh
|
||||
kubectl apply -f https://raw.githubusercontent.com/<org>/egress-proxies-operator/<tag or branch>/dist/install.yaml
|
||||
```
|
||||
|
||||
### By providing a Helm Chart
|
||||
|
||||
1. Build the chart using the optional helm plugin
|
||||
|
||||
```sh
|
||||
kubebuilder edit --plugins=helm/v2-alpha
|
||||
```
|
||||
|
||||
2. See that a chart was generated under 'dist/chart', and users
|
||||
can obtain this solution from there.
|
||||
|
||||
**NOTE:** If you change the project, you need to update the Helm Chart
|
||||
using the same command above to sync the latest changes. Furthermore,
|
||||
if you create webhooks, you need to use the above command with
|
||||
the '--force' flag and manually ensure that any custom configuration
|
||||
previously added to 'dist/chart/values.yaml' or 'dist/chart/manager/manager.yaml'
|
||||
is manually re-applied afterwards.
|
||||
|
||||
## Contributing
|
||||
// TODO(user): Add detailed information on how you would like others to contribute to this project
|
||||
|
||||
**NOTE:** Run `make help` for more information on all potential `make` targets
|
||||
|
||||
More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html)
|
||||
|
||||
## License
|
||||
|
||||
Copyright 2026.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
`make run-dev` is for iterating on the operator itself: provisioning,
|
||||
replacement, the discovery API, and External proxies all work from your
|
||||
laptop. Health checks against in-cluster pods do **not** (see the
|
||||
quickstart note) — use the in-cluster deploy to see a kubernetes-provider
|
||||
proxy go `Ready`.
|
||||
|
||||
Project layout, reconcile-loop diagrams, and the decision log are in
|
||||
[docs/architecture.md](docs/architecture.md); the build history is in
|
||||
[docs/plans-executions/](docs/plans-executions/).
|
||||
|
||||
@@ -61,6 +61,12 @@ const (
|
||||
// provision. A mismatch against the freshly computed hash means the VM
|
||||
// must be replaced.
|
||||
AnnotationSpecHash = "crawl.example.com/spec-hash"
|
||||
|
||||
// LabelCloudInit must be set (to "true") on every Secret referenced by
|
||||
// spec.cloudInit.secretRef: the manager's cache only holds Secrets
|
||||
// carrying this label, so an unlabelled Secret is invisible to the
|
||||
// operator — both to the resolve step and to the rotation watch.
|
||||
LabelCloudInit = "crawl.example.com/cloud-init"
|
||||
)
|
||||
|
||||
// Defaults, applied both by CRD structural defaulting (kubebuilder:default
|
||||
|
||||
182
cmd/main.go
182
cmd/main.go
@@ -14,28 +14,48 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package main is the composition root: it loads the provider config (fail
|
||||
// fast), assembles the provider registry, and wires the reconciler, health
|
||||
// engine, lease store, discovery API, orphan GC, and metrics onto one
|
||||
// controller-runtime manager.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"flag"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
|
||||
// to ensure that exec-entrypoint and run can make use of them.
|
||||
_ "k8s.io/client-go/plugin/pkg/client/auth"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/cache"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/healthz"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log/zap"
|
||||
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
|
||||
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
|
||||
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/controller"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/discovery"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/gc"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/metrics"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
|
||||
// +kubebuilder:scaffold:imports
|
||||
)
|
||||
|
||||
@@ -60,6 +80,15 @@ func main() {
|
||||
var secureMetrics bool
|
||||
var enableHTTP2 bool
|
||||
var tlsOpts []func(*tls.Config)
|
||||
|
||||
var providersConfig string
|
||||
var discoveryAddr string
|
||||
var proxyNamespace string
|
||||
var healthWorkers int
|
||||
var gcInterval, gcMinAge time.Duration
|
||||
var gcAllowNamespaced bool
|
||||
var leaseCooldown, maxLeaseTTL time.Duration
|
||||
|
||||
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
|
||||
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
|
||||
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
|
||||
@@ -74,6 +103,28 @@ func main() {
|
||||
flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
|
||||
flag.BoolVar(&enableHTTP2, "enable-http2", false,
|
||||
"If set, HTTP/2 will be enabled for the metrics server")
|
||||
|
||||
flag.StringVar(&providersConfig, "providers-config", "",
|
||||
"Path to the providers config YAML. Required.")
|
||||
flag.StringVar(&discoveryAddr, "discovery-addr", ":8090",
|
||||
"Listen address of the discovery/lease HTTP API.")
|
||||
flag.StringVar(&proxyNamespace, "proxy-namespace", "",
|
||||
"Restrict the manager's cache to one namespace. Empty watches all namespaces. "+
|
||||
"Restricting also disables orphan GC unless --gc-allow-namespaced is set.")
|
||||
flag.IntVar(&healthWorkers, "health-workers", 8,
|
||||
"Number of concurrent health-probe workers.")
|
||||
flag.DurationVar(&gcInterval, "gc-interval", 10*time.Minute,
|
||||
"Interval between orphan GC sweeps.")
|
||||
flag.DurationVar(&gcMinAge, "gc-min-age", 10*time.Minute,
|
||||
"Minimum instance age before orphan GC may delete it.")
|
||||
flag.BoolVar(&gcAllowNamespaced, "gc-allow-namespaced", false,
|
||||
"Allow orphan GC to run although the cache is namespace-restricted. Dangerous: proxies "+
|
||||
"outside the namespace count as orphans and their instances get deleted.")
|
||||
flag.DurationVar(&leaseCooldown, "lease-cooldown", 15*time.Minute,
|
||||
"How long a reported proxy/target pair is excluded from lease selection.")
|
||||
flag.DurationVar(&maxLeaseTTL, "max-lease-ttl", time.Hour,
|
||||
"Maximum lease TTL a client may request.")
|
||||
|
||||
opts := zap.Options{
|
||||
Development: true,
|
||||
}
|
||||
@@ -81,6 +132,31 @@ func main() {
|
||||
flag.Parse()
|
||||
|
||||
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
|
||||
ctx := ctrl.SetupSignalHandler()
|
||||
|
||||
// Providers load first and fail fast: a manager that comes up without
|
||||
// its backends would just convert every Proxy into an error loop.
|
||||
if providersConfig == "" {
|
||||
setupLog.Error(nil, "--providers-config is required")
|
||||
os.Exit(1)
|
||||
}
|
||||
cfg, err := provider.LoadConfigFile(providersConfig)
|
||||
if err != nil {
|
||||
setupLog.Error(err, "Failed to load providers config", "path", providersConfig)
|
||||
os.Exit(1)
|
||||
}
|
||||
providers, err := registry.Build(ctx, cfg, map[string]registry.Constructor{
|
||||
"kubernetes": kubernetes.New,
|
||||
"gcp": gcp.New,
|
||||
})
|
||||
if err != nil {
|
||||
setupLog.Error(err, "Failed to build providers")
|
||||
os.Exit(1)
|
||||
}
|
||||
m := metrics.New()
|
||||
for name, p := range providers {
|
||||
providers[name] = provider.WithMetrics(name, p, m)
|
||||
}
|
||||
|
||||
// if the enable-http2 flag is false (the default), http/2 should be disabled
|
||||
// due to its vulnerabilities. More specifically, disabling http/2 will
|
||||
@@ -128,32 +204,91 @@ func main() {
|
||||
metricsServerOptions.KeyName = metricsCertKey
|
||||
}
|
||||
|
||||
// The Secret cache is restricted to labelled cloud-init Secrets: the
|
||||
// operator has cluster-wide Secret read RBAC, and without the label
|
||||
// selector it would cache every Secret in scope.
|
||||
cacheOpts := cache.Options{
|
||||
ByObject: map[client.Object]cache.ByObject{
|
||||
&corev1.Secret{}: {
|
||||
Label: labels.SelectorFromSet(labels.Set{crawlv1alpha1.LabelCloudInit: "true"}),
|
||||
},
|
||||
},
|
||||
}
|
||||
if proxyNamespace != "" {
|
||||
cacheOpts.DefaultNamespaces = map[string]cache.Config{proxyNamespace: {}}
|
||||
}
|
||||
|
||||
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
|
||||
Scheme: scheme,
|
||||
Metrics: metricsServerOptions,
|
||||
HealthProbeBindAddress: probeAddr,
|
||||
Cache: cacheOpts,
|
||||
LeaderElection: enableLeaderElection,
|
||||
LeaderElectionID: "b47711d1.example.com",
|
||||
// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
|
||||
// when the Manager ends. This requires the binary to immediately end when the
|
||||
// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
|
||||
// speeds up voluntary leader transitions as the new leader don't have to wait
|
||||
// LeaseDuration time first.
|
||||
//
|
||||
// In the default scaffold provided, the program ends immediately after
|
||||
// the manager stops, so would be fine to enable this option. However,
|
||||
// if you are doing or is intended to do any operation such as perform cleanups
|
||||
// after the manager stops then its usage might be unsafe.
|
||||
// LeaderElectionReleaseOnCancel: true,
|
||||
})
|
||||
if err != nil {
|
||||
setupLog.Error(err, "Failed to start manager")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
store := lease.NewStore(leaseCooldown)
|
||||
if err := mgr.Add(store); err != nil {
|
||||
setupLog.Error(err, "Failed to add lease store")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
engine := health.NewEngine(mgr.GetClient())
|
||||
engine.Workers = healthWorkers
|
||||
engine.Metrics = m
|
||||
if err := mgr.Add(engine); err != nil {
|
||||
setupLog.Error(err, "Failed to add health engine")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := mgr.Add(&discovery.Server{
|
||||
Reader: mgr.GetClient(),
|
||||
Store: store,
|
||||
Addr: discoveryAddr,
|
||||
Token: os.Getenv("DISCOVERY_TOKEN"),
|
||||
MaxLeaseTTL: maxLeaseTTL,
|
||||
Metrics: m,
|
||||
}); err != nil {
|
||||
setupLog.Error(err, "Failed to add discovery server")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := mgr.Add(&gc.Sweeper{
|
||||
Reader: mgr.GetClient(),
|
||||
Providers: providers,
|
||||
Interval: gcInterval,
|
||||
MinAge: gcMinAge,
|
||||
NamespaceRestricted: proxyNamespace != "",
|
||||
AllowNamespaced: gcAllowNamespaced,
|
||||
}); err != nil {
|
||||
setupLog.Error(err, "Failed to add orphan GC")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := m.Register(ctrlmetrics.Registry,
|
||||
proxyPhaseCounts(mgr.GetClient()),
|
||||
func() int {
|
||||
total := 0
|
||||
for _, n := range store.Counts() {
|
||||
total += n
|
||||
}
|
||||
return total
|
||||
},
|
||||
); err != nil {
|
||||
setupLog.Error(err, "Failed to register metrics")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := (&controller.ProxyReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Scheme: mgr.GetScheme(),
|
||||
Providers: providers,
|
||||
Health: engine,
|
||||
HealthEvents: engine.Events,
|
||||
}).SetupWithManager(mgr); err != nil {
|
||||
setupLog.Error(err, "Failed to create controller", "controller", "proxy")
|
||||
os.Exit(1)
|
||||
@@ -170,8 +305,31 @@ func main() {
|
||||
}
|
||||
|
||||
setupLog.Info("Starting manager")
|
||||
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
|
||||
if err := mgr.Start(ctx); err != nil {
|
||||
setupLog.Error(err, "Failed to run manager")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// proxyPhaseCounts reads phase counts from the cache at scrape time. Before
|
||||
// the cache has synced (or on any list error) it reports nothing rather
|
||||
// than something wrong.
|
||||
func proxyPhaseCounts(c client.Reader) func() map[string]int {
|
||||
return func() map[string]int {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
var list crawlv1alpha1.ProxyList
|
||||
if err := c.List(ctx, &list); err != nil {
|
||||
return nil
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for i := range list.Items {
|
||||
phase := string(list.Items[i].Status.Phase)
|
||||
if phase == "" {
|
||||
phase = string(crawlv1alpha1.PhasePending)
|
||||
}
|
||||
counts[phase]++
|
||||
}
|
||||
return counts
|
||||
}
|
||||
}
|
||||
|
||||
18
config/default/discovery_service.yaml
Normal file
18
config/default/discovery_service.yaml
Normal file
@@ -0,0 +1,18 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
labels:
|
||||
control-plane: controller-manager
|
||||
app.kubernetes.io/name: egress-proxies-operator
|
||||
app.kubernetes.io/managed-by: kustomize
|
||||
name: controller-manager-discovery-service
|
||||
namespace: system
|
||||
spec:
|
||||
ports:
|
||||
- name: discovery
|
||||
port: 8090
|
||||
protocol: TCP
|
||||
targetPort: discovery
|
||||
selector:
|
||||
control-plane: controller-manager
|
||||
app.kubernetes.io/name: egress-proxies-operator
|
||||
@@ -22,6 +22,8 @@ resources:
|
||||
#- ../prometheus
|
||||
# [METRICS] Expose the controller manager metrics service.
|
||||
- metrics_service.yaml
|
||||
# Expose the discovery/lease HTTP API inside the cluster.
|
||||
- discovery_service.yaml
|
||||
|
||||
# Uncomment the patches line if you enable Metrics
|
||||
patches:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
resources:
|
||||
- manager.yaml
|
||||
- providers_config.yaml
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
images:
|
||||
|
||||
@@ -63,12 +63,28 @@ spec:
|
||||
args:
|
||||
- --leader-elect
|
||||
- --health-probe-bind-address=:8081
|
||||
- --providers-config=/etc/proxy-operator/providers.yaml
|
||||
env:
|
||||
# Bearer token for the discovery API. Optional: without the
|
||||
# Secret the API serves unauthenticated (with a loud warning).
|
||||
# Create it with:
|
||||
# kubectl -n egress-proxies-operator-system create secret \
|
||||
# generic discovery-token --from-literal=token=<your-token>
|
||||
- name: DISCOVERY_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: discovery-token
|
||||
key: token
|
||||
optional: true
|
||||
image: controller:latest
|
||||
name: manager
|
||||
ports:
|
||||
- containerPort: 8081
|
||||
name: health
|
||||
protocol: TCP
|
||||
- containerPort: 8090
|
||||
name: discovery
|
||||
protocol: TCP
|
||||
securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
allowPrivilegeEscalation: false
|
||||
@@ -96,7 +112,13 @@ spec:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
volumeMounts: []
|
||||
volumes: []
|
||||
volumeMounts:
|
||||
- name: providers-config
|
||||
mountPath: /etc/proxy-operator
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: providers-config
|
||||
configMap:
|
||||
name: providers-config
|
||||
serviceAccountName: controller-manager
|
||||
terminationGracePeriodSeconds: 10
|
||||
|
||||
17
config/manager/providers_config.yaml
Normal file
17
config/manager/providers_config.yaml
Normal file
@@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: providers-config
|
||||
namespace: system
|
||||
labels:
|
||||
app.kubernetes.io/name: egress-proxies-operator
|
||||
app.kubernetes.io/managed-by: kustomize
|
||||
data:
|
||||
# Mounted at /etc/proxy-operator/providers.yaml (--providers-config).
|
||||
# The default ships only the kubernetes-pod provider so the operator runs
|
||||
# out of the box; add gcp entries (type: gcp, gcp.project: ...) for real
|
||||
# egress fleets — see config/samples/providers-config.yaml.
|
||||
providers.yaml: |
|
||||
providers:
|
||||
- name: kubernetes
|
||||
type: kubernetes
|
||||
@@ -4,6 +4,16 @@ kind: ClusterRole
|
||||
metadata:
|
||||
name: manager-role
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
verbs:
|
||||
- create
|
||||
- delete
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
apiVersion: crawl.example.com/v1alpha1
|
||||
kind: Proxy
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: egress-proxies-operator
|
||||
app.kubernetes.io/managed-by: kustomize
|
||||
name: proxy-sample
|
||||
spec:
|
||||
# TODO(user): Add fields here
|
||||
@@ -1,4 +1,8 @@
|
||||
## Append samples of your project ##
|
||||
# providers-config.yaml is deliberately absent: it is a sample
|
||||
# --providers-config file, not a Kubernetes manifest.
|
||||
resources:
|
||||
- crawl_v1alpha1_proxy.yaml
|
||||
- proxy_kubernetes.yaml
|
||||
- proxy_gcp.yaml
|
||||
- proxy_external.yaml
|
||||
# +kubebuilder:scaffold:manifestskustomizesamples
|
||||
|
||||
21
config/samples/providers-config.yaml
Normal file
21
config/samples/providers-config.yaml
Normal file
@@ -0,0 +1,21 @@
|
||||
# Sample --providers-config file (not a Kubernetes manifest). In-cluster
|
||||
# this content lives in the providers-config ConfigMap
|
||||
# (config/manager/providers_config.yaml); for `make run-dev` a
|
||||
# kubernetes-only variant is at hack/providers-dev.yaml.
|
||||
#
|
||||
# Named provider instances: "gcp-eu" and "gcp-us" are two configs of the
|
||||
# same type. spec.provider on a Proxy refers to the name, not the type.
|
||||
providers:
|
||||
- name: kubernetes
|
||||
type: kubernetes
|
||||
# kubernetes:
|
||||
# image: ubuntu/squid:6.6-24.04_edge # the default
|
||||
- name: gcp-eu
|
||||
type: gcp
|
||||
gcp:
|
||||
project: my-project
|
||||
# network: default # VPC network name
|
||||
# networkTag: proxy-operator # firewall tag on created instances
|
||||
# diskSizeGb: 10
|
||||
# auth: Application Default Credentials (workload identity
|
||||
# in-cluster, gcloud ADC locally). No key-file plumbing.
|
||||
15
config/samples/proxy_external.yaml
Normal file
15
config/samples/proxy_external.yaml
Normal file
@@ -0,0 +1,15 @@
|
||||
# An External proxy: the VM exists outside the operator's control; the
|
||||
# operator only tracks and health-checks it through the endpoint. No
|
||||
# finalizer, no provider calls, and deleting the CR touches nothing.
|
||||
apiVersion: crawl.example.com/v1alpha1
|
||||
kind: Proxy
|
||||
metadata:
|
||||
name: proxy-external-sample
|
||||
spec:
|
||||
mode: External
|
||||
endpoint:
|
||||
host: 203.0.113.7
|
||||
port: 3128
|
||||
attributes:
|
||||
geo: eu
|
||||
purpose: crawl
|
||||
36
config/samples/proxy_gcp.yaml
Normal file
36
config/samples/proxy_gcp.yaml
Normal file
@@ -0,0 +1,36 @@
|
||||
# A Managed proxy backed by GCP: the operator creates a VM with an
|
||||
# ephemeral external IP and installs Squid via cloud-init. Requires a
|
||||
# providers-config entry named "gcp-eu" (see providers-config.yaml) and
|
||||
# Application Default Credentials with compute.instanceAdmin.v1.
|
||||
#
|
||||
# All three placement fields are required for GCP; the operator sets the
|
||||
# Proxy to Failed with a message naming any missing one.
|
||||
apiVersion: crawl.example.com/v1alpha1
|
||||
kind: Proxy
|
||||
metadata:
|
||||
name: proxy-gcp-sample
|
||||
spec:
|
||||
mode: Managed
|
||||
provider: gcp-eu
|
||||
placement:
|
||||
zone: europe-west1-b
|
||||
machineType: e2-micro
|
||||
image: projects/debian-cloud/global/images/family/debian-12
|
||||
port: 3128
|
||||
cloudInit:
|
||||
inline: |
|
||||
#cloud-config
|
||||
packages:
|
||||
- squid
|
||||
write_files:
|
||||
- path: /etc/squid/conf.d/proxy-operator.conf
|
||||
content: |
|
||||
http_port 3128
|
||||
http_access allow all
|
||||
via off
|
||||
forwarded_for off
|
||||
runcmd:
|
||||
- systemctl restart squid
|
||||
attributes:
|
||||
geo: eu
|
||||
purpose: crawl
|
||||
14
config/samples/proxy_kubernetes.yaml
Normal file
14
config/samples/proxy_kubernetes.yaml
Normal file
@@ -0,0 +1,14 @@
|
||||
# A Managed proxy backed by the kubernetes-pod provider: the operator runs
|
||||
# a real Squid pod in this cluster. This is the local-dev/CI sample — pods
|
||||
# share the cluster's egress IP, so it exercises the full lifecycle but
|
||||
# does not provide a distinct egress path (use the gcp provider for that).
|
||||
apiVersion: crawl.example.com/v1alpha1
|
||||
kind: Proxy
|
||||
metadata:
|
||||
name: proxy-kubernetes-sample
|
||||
spec:
|
||||
mode: Managed
|
||||
provider: kubernetes
|
||||
attributes:
|
||||
geo: local
|
||||
purpose: crawl
|
||||
@@ -1,10 +1,20 @@
|
||||
# Architecture
|
||||
|
||||
> **Status:** the operator is built through Step 5 (health engine) of
|
||||
> [docs/plans/2026-08-07-1747-proxy-operator.md](plans/2026-08-07-1747-proxy-operator.md).
|
||||
> This document currently covers the event/reconcile flow; the components
|
||||
> table and the Decisions section arrive with Step 10, and the diagrams
|
||||
> below grow as the lease store, discovery API, and orphan GC land.
|
||||
## Components
|
||||
|
||||
| Component | Package | Runs as | Leader-elected | Role |
|
||||
|---|---|---|---|---|
|
||||
| Proxy CRD + helpers | `api/v1alpha1` | types | — | `Proxy` spec/status, CEL validation, defaulting, pure helpers |
|
||||
| Reconciler | `internal/controller` | controller | yes (with the manager) | the state machine: provision, replace, delete, represent health |
|
||||
| Provider contract | `internal/provider` | library | — | `Provider` interface, error taxonomy, deterministic naming, config, metrics decorator |
|
||||
| kubernetes provider | `internal/provider/kubernetes` | library | — | real Squid pods in this cluster (local dev/CI) |
|
||||
| gcp provider | `internal/provider/gcp` | library | — | Compute Engine VMs, four API calls, fire-and-forget ops |
|
||||
| Health engine | `internal/health` | Runnable | yes | through-the-proxy probes, thresholds, transition events |
|
||||
| Lease store | `internal/lease` | Runnable (expiry sweep) | no | in-memory leases + cooldowns, single mutex |
|
||||
| Discovery API | `internal/discovery` | Runnable | no | HTTP list/lease/release/report on `:8090` |
|
||||
| Orphan GC | `internal/gc` | Runnable | yes | deletes tagged instances whose CR is gone |
|
||||
| Metrics | `internal/metrics` | library | — | explicit registration, scrape-time collectors |
|
||||
| Composition root | `cmd/main.go` | binary | — | flags, provider registry, wires everything onto one manager |
|
||||
|
||||
## Event flow: cluster events → reconciler functions
|
||||
|
||||
@@ -117,20 +127,32 @@ reconcileDelete(ctx, p) reconcileExternal(ctx, p)
|
||||
──► RequeueAfter: DeletionPoll (poll until gone)
|
||||
```
|
||||
|
||||
### 5. What provider calls do back in the cluster (kubernetes pod provider)
|
||||
### 5. What provider calls do in the outside world
|
||||
|
||||
```text
|
||||
kubernetes pod provider (internal/provider/kubernetes/)
|
||||
prov.Create ──► buildPod (pure) ──► client.Create(corev1.Pod) ─┐ these cause Pod events,
|
||||
prov.Get ──► client.Get(Pod) → phase/IP → InstanceState │ but the operator does NOT
|
||||
prov.Delete ──► client.Delete(Pod, tolerate NotFound) │ watch Pods — it observes
|
||||
prov.ListByTag ─► client.List(Pods by labels, all namespaces) ─┘ them by polling prov.Get
|
||||
on each RequeueAfter tick
|
||||
|
||||
gcp provider (internal/provider/gcp/) — instances.{Insert,Get,Delete,AggregatedList}, nothing else
|
||||
prov.Create ──► buildInsertRequest (pure) ──► instances.Insert ─┐ fire-and-forget:
|
||||
409 alreadyExists = success (idempotent retry) │ Operation.Wait is never
|
||||
prov.Get ──► instances.Get → status/NatIP → InstanceState │ called; readiness is
|
||||
RUNNING without NatIP = still Provisioning │ discovered by Get polls,
|
||||
prov.Delete ──► instances.Delete (404 = success) │ exactly like the pod
|
||||
prov.ListByTag ─► AggregatedList(label filter, ─┘ provider
|
||||
ReturnPartialSuccess: true)
|
||||
providerID = zones/<zone>/instances/<name> — zone-qualified, so Get/Delete
|
||||
stay correct even mid-replacement after a zone edit
|
||||
```
|
||||
|
||||
The reconciler never watches provider-side resources (Pods now, GCP VMs
|
||||
later). All instance-state observation is poll-based through the
|
||||
`Provider` interface, so the same flow works identically for a cloud API
|
||||
that has no watch mechanism at all.
|
||||
The reconciler never watches provider-side resources (Pods or GCP VMs).
|
||||
All instance-state observation is poll-based through the `Provider`
|
||||
interface, so the same flow works identically for a cloud API that has no
|
||||
watch mechanism at all.
|
||||
|
||||
### 6. Health engine (`internal/health/`) — probes and transitions
|
||||
|
||||
@@ -183,3 +205,200 @@ Consequence worth knowing: `status.lastHealthCheckTime` is the time of the
|
||||
last *status-affecting* probe, not the most recent probe — suppressed
|
||||
probes deliberately never write status. True probe recency will live in
|
||||
metrics (Step 9).
|
||||
|
||||
### 7. Discovery + lease API (`internal/discovery/`, `internal/lease/`)
|
||||
|
||||
HTTP-driven, not cluster-event-driven: crawler clients call in; the only
|
||||
Kubernetes interaction is reading Proxies from the manager's cache. The
|
||||
server is a non-leader-elected Runnable (all replicas would serve, but the
|
||||
deployment ships `replicas: 1` because lease state is per-process — an
|
||||
operator restart drops all leases and cooldowns, a documented caveat).
|
||||
|
||||
```text
|
||||
crawler client
|
||||
│ Authorization: Bearer $DISCOVERY_TOKEN (empty token = auth disabled, loud startup warning)
|
||||
▼
|
||||
Server.handler() middleware, outermost first (server.go)
|
||||
recover → request-log → MaxBytesReader(64KiB) → bearer auth (constant-time; /healthz exempt)
|
||||
│
|
||||
├─ GET /healthz ──► 200 ok (unauthenticated)
|
||||
│
|
||||
├─ GET /v1/proxies?attr.k=v&healthy=true (handlers.go)
|
||||
│ Reader.List(Proxies) ── manager cache
|
||||
│ filter: attributes equality + Healthy condition
|
||||
│ + Store.Counts() for activeLeases
|
||||
│ ──► 200 {"proxies":[...], "count":N} (empty list is 200, not 404)
|
||||
│
|
||||
├─ POST /v1/leases {"selector":{...},"ttlSeconds":300,"target":"..."}
|
||||
│ Reader.List → filter selector; unhealthy matches counted, not offered
|
||||
│ Store.Acquire(healthy candidates, target, ttl) ── one lock: select+insert
|
||||
│ │ selection: fewest active leases, then latency, then name
|
||||
│ ├─ granted ──► 201 {leaseID, proxy:{...}, expiresAt, ttlSeconds}
|
||||
│ └─ ErrNoMatch ──► 409 {"error":"no_match", considered, atCapacity,
|
||||
│ inCooldown, unhealthy}
|
||||
│
|
||||
├─ DELETE /v1/leases/{id} ──► Store.Release ──► always 204 (idempotent)
|
||||
│
|
||||
└─ POST /v1/leases/{id}/report {"result":"ok|rate_limited|banned","target":"..."}
|
||||
Store.Report ── rate_limited/banned ⇒ cooldown[{proxy,target}] for
|
||||
│ CooldownWindow (target falls back: report → lease → global)
|
||||
├─ 204 │ 400 invalid_result │ 404 unknown_lease
|
||||
└─ an expired lease still resolves for CooldownWindow past its TTL —
|
||||
a late report lands exactly when the proxy is being rate-limited
|
||||
|
||||
Store.Start(ctx) ── manager Runnable, NOT leader-elected: sweeps expired
|
||||
leases + cooldowns; correctness never depends on the
|
||||
sweep (every read checks ExpiresAt against the clock)
|
||||
```
|
||||
|
||||
### 8. Orphan GC (`internal/gc/`) — the crash-safety net
|
||||
|
||||
Timer-driven, leader-elected (destructive ⇒ single writer). Exists for the
|
||||
one gap the reconciler cannot close alone: a crash after a provider Create
|
||||
but before the status write that records the instance.
|
||||
|
||||
```text
|
||||
Sweeper.Start(ctx) ── refuses to run when the cache is namespace-
|
||||
│ restricted unless --gc-allow-namespaced is explicit
|
||||
│ (an incomplete live set would "orphan" live VMs)
|
||||
└─ every Interval (10m; first sweep a full interval after start):
|
||||
sweep(ctx)
|
||||
│ Reader.List(Proxies) → live UID set
|
||||
│ List fails → skip the whole sweep (never guess)
|
||||
│ a CR with deletionTimestamp still counts as LIVE — its
|
||||
│ finalizer owns that deletion; GC racing it double-deletes
|
||||
└ per provider: ListByTag
|
||||
│ error → log, continue with the next provider
|
||||
└ delete only when ALL hold:
|
||||
has the proxy-operator-uid label (ownership proof)
|
||||
older than MinAge (10m) (not mid-create)
|
||||
UID matches no existing CR (truly orphaned)
|
||||
each kill logged loudly with provider, providerID, UID
|
||||
```
|
||||
|
||||
### 9. Metrics (`internal/metrics/`)
|
||||
|
||||
Registered explicitly from `cmd/main.go` (no `init()`; tests use fresh
|
||||
registries). Two kinds:
|
||||
|
||||
- **Scrape-time collectors** — `proxy_operator_proxies{phase}` and
|
||||
`proxy_operator_leases_active` read the cache / lease store at every
|
||||
scrape; reconcile-incremented gauges inevitably drift and leak series.
|
||||
- **Fed vectors** — `healthcheck_duration_seconds{proxy}` and
|
||||
`healthcheck_failures_total{proxy}` observe EVERY probe (status writes
|
||||
are transition-only; metrics carry the high-frequency signal), and the
|
||||
health engine deletes a proxy's series when it prunes its state;
|
||||
`lease_requests_total{outcome}` from the discovery handlers;
|
||||
`provider_requests_total{provider,op,result}` from the
|
||||
`provider.WithMetrics` decorator — the one place `Class()` is called
|
||||
purely for observability.
|
||||
|
||||
Each consuming package defines its own small recorder interface
|
||||
(`health.ProbeMetrics`, `discovery.LeaseMetrics`, `provider.RequestRecorder`);
|
||||
`metrics.Metrics` satisfies all of them structurally, so no package other
|
||||
than `cmd/main.go` imports the metrics package.
|
||||
|
||||
## Decisions
|
||||
|
||||
Judgment calls the spec left open, and deliberate deviations — recorded so
|
||||
they read as choices, not accidents. Chronological by build step.
|
||||
|
||||
- **Registry takes its constructor map as a parameter** instead of holding
|
||||
a package-level map: avoids the provider⇄registry import cycle and puts
|
||||
the wiring at the composition root, where it is visible.
|
||||
- **Mock provider replaced by the kubernetes-pod provider** (user
|
||||
decision, mid-build): a simulated in-memory provider was too far from
|
||||
the real system to build confidence in. Local dev/CI now runs real
|
||||
`ubuntu/squid` pods (Canonical's actively maintained image, verified
|
||||
50M+ pulls, pinned tag) in the operator's own cluster. Trade-off
|
||||
accepted: envtest has no kubelet, so end-to-end proof lives in the kind
|
||||
quickstart, and cluster pods share one egress IP — distinct egress
|
||||
paths remain the GCP provider's job.
|
||||
- **`RequeueAfter: RequeueNow` instead of the plan's `Requeue: true`:**
|
||||
`ctrl.Result.Requeue` is deprecated in controller-runtime v0.24; a fifth
|
||||
configurable interval (default 1s) keeps identical semantics and stays
|
||||
shrinkable in tests.
|
||||
- **Quota exhaustion is a wait, not a failure:** `ErrQuotaExceeded` sets a
|
||||
condition and requeues slowly (5m) with a nil error — off the backoff
|
||||
curve, out of the error log, and never `phase: Failed`. Only
|
||||
`ErrPermanent` latches Failed, keyed to the generation so a spec edit
|
||||
auto-recovers.
|
||||
- **The finalizer path never latches permanent failures:** a permanent
|
||||
error during deletion keeps retrying visibly instead — latching there
|
||||
would wedge the object forever with no path out but manual finalizer
|
||||
surgery.
|
||||
- **Health transitions travel reconciler-ward over a channel**
|
||||
(`source.Channel`), not direct status patches: `phase` derives from both
|
||||
provisioning and health, so two status writers would race and flap. One
|
||||
writer of status; the engine owns health *state*, the reconciler its
|
||||
*representation*; write-only-on-transition falls out for free.
|
||||
- **Health state seeds from the existing Healthy condition on leader
|
||||
handover** (verdict kept, counters zeroed, first probe jittered), so a
|
||||
healthy fleet doesn't flap to Unknown on restart — but a real
|
||||
transition still needs a full threshold run. A never-probed proxy skips
|
||||
the jitter and probes on the next tick: startup spread matters for
|
||||
restarts, not for a single new proxy.
|
||||
- **Latency suppression is `max(20ms, 50%)` + a 60s rate limit, and only
|
||||
while the verdict is healthy.** The spec's bare ">50% change" is
|
||||
undefined at 0 and lets a proxy jittering 40↔61ms write status forever;
|
||||
the healthy-only guard (found by test) stops a below-threshold success
|
||||
streak from emitting latency updates for a proxy still reported
|
||||
unhealthy. Consequence: `status.lastHealthCheckTime` means "last
|
||||
status-affecting probe" — true probe recency is in the metrics.
|
||||
- **Deterministic instance names** are `proxy-` + 16 chars of
|
||||
base32(SHA-256(CR UID)): legal for both GCP (`[a-z2-7]` ⊂ `[-a-z0-9]`,
|
||||
22 ≤ 63 chars) and Pod names, 80 bits against birthday collisions at a
|
||||
fleet of tens. The replacement VM therefore has the *same name* as the
|
||||
one being deleted — which is why replacement polls to NotFound before
|
||||
recreating instead of racing a 409.
|
||||
- **`banned` and `rate_limited` share one cooldown window:** a second
|
||||
duration knob the spec doesn't ask for; the report's semantic
|
||||
difference is preserved in the API but not the store.
|
||||
- **Report targets fall back report → lease → global**, so a client that
|
||||
leased with a target can't accidentally poison the proxy's global pool
|
||||
by omitting the target in its report.
|
||||
- **The 409 body's `considered` counts unhealthy matches too** (the store
|
||||
only ever sees healthy candidates): `considered = atCapacity +
|
||||
inCooldown + unhealthy + eligible-but-outranked`, keeping the numbers
|
||||
additive for a human debugging "why no proxy?".
|
||||
- **TTLs above `--max-lease-ttl` are a 400, not a silent clamp** — a
|
||||
client asking for a week should find out.
|
||||
- **Discovery is not leader-elected and ships `replicas: 1`:** caches
|
||||
start before non-leader-election runnables (verified in
|
||||
controller-runtime's ordering), and a leader-elected server would leave
|
||||
non-leader replicas as broken Service endpoints. One replica because
|
||||
lease state is per-process.
|
||||
- **GCP `Create` requires zone, machineType, and image** and fails
|
||||
`ErrPermanent` naming the missing field — inventing machine-type
|
||||
defaults would silently create billable VMs of arbitrary shape.
|
||||
- **Unknown GCP instance statuses map to `Stopped`:** the reconciler's
|
||||
answer to Stopped is delete-and-recreate, the always-safe move for
|
||||
cattle when the API grows a new state.
|
||||
- **Kubernetes 403s classify as `ErrPermanent`** even though quota
|
||||
exhaustion also surfaces as 403 (indistinguishable from RBAC denial in
|
||||
`apierrors`): not hammering an API server that may never allow the
|
||||
request is the safer default; a real ResourceQuota 403 forgoes the
|
||||
gentler quota backoff. Documented at the classification site.
|
||||
- **GC kills log at Info with a `WARNING:` prefix** — logr has no Warn
|
||||
level; the plan's "log at Warn" is met in spirit with provider,
|
||||
providerID, and UID always attached. Same convention as the
|
||||
discovery server's empty-token warning.
|
||||
- **GC trusts only provable orphans:** instances without the UID label
|
||||
are never deleted, a CR with a deletionTimestamp still counts as live
|
||||
(its finalizer owns that deletion), and an unreadable Proxy list skips
|
||||
the whole sweep. The namespace guard refuses to sweep a
|
||||
namespace-restricted cache without `--gc-allow-namespaced`.
|
||||
- **Cloud-init Secrets must carry `crawl.example.com/cloud-init: "true"`:**
|
||||
the manager caches only labelled Secrets (the operator holds
|
||||
cluster-wide Secret read RBAC — an unrestricted cache would hold every
|
||||
Secret in scope). Unlabelled referenced Secrets are invisible by
|
||||
construction, surfacing as `CloudInitError`.
|
||||
- **Events RBAC from the plan is omitted:** nothing wires an
|
||||
EventRecorder in the prototype, and granting verbs nothing uses would
|
||||
be RBAC lint noise. Add the marker together with the recorder if events
|
||||
land later.
|
||||
- **logr, not slog, inside controller paths:** the repo convention says
|
||||
`slog`, but `log.FromContext(ctx)` hands controller-runtime's logr
|
||||
logger to everything running under the manager — fighting that would
|
||||
mean two logging systems in one process. Noted as a deviation rather
|
||||
than silently ignored.
|
||||
|
||||
@@ -10,13 +10,13 @@ Pairs with [docs/plans/2026-08-07-1747-proxy-operator.md](../plans/2026-08-07-17
|
||||
- [x] Step 3 — Kubernetes pod provider (`internal/provider/kubernetes/`; first built as an in-memory mock, then replaced — see the two Step 3 sections below)
|
||||
- [x] Step 4 — Reconciler (`internal/controller/`)
|
||||
- [x] Step 5 — Health engine (`internal/health/`)
|
||||
- [ ] Step 6 — Lease store (`internal/lease/`)
|
||||
- [ ] Step 7 — Discovery API (`internal/discovery/`)
|
||||
- [ ] Step 8 — GCP provider (`internal/provider/gcp/`)
|
||||
- [ ] Step 9 — Orphan GC + metrics
|
||||
- [ ] Step 10 — Wiring, config, docs
|
||||
- [ ] Step 11 — Tests
|
||||
- [ ] Verification (vet/test/kind e2e) + commit, push, open MR
|
||||
- [x] Step 6 — Lease store (`internal/lease/`)
|
||||
- [x] Step 7 — Discovery API (`internal/discovery/`)
|
||||
- [x] Step 8 — GCP provider (`internal/provider/gcp/`)
|
||||
- [x] Step 9 — Orphan GC + metrics
|
||||
- [x] Step 10 — Wiring, config, docs
|
||||
- [x] Step 11 — Tests
|
||||
- [x] Verification (vet/test/kind e2e) + commit, push, open MR
|
||||
|
||||
## Step 0 — Branch and scaffold
|
||||
|
||||
@@ -694,3 +694,438 @@ engine deliberately knows nothing about conditions except reading one at
|
||||
seed time, keeping the state/representation split honest. The
|
||||
`hint`-driven `wg.Go` idiom (Go 1.25+) replaced the classic
|
||||
`wg.Add/defer wg.Done` in the worker pool.
|
||||
|
||||
## Step 6 — Lease store (`internal/lease/`)
|
||||
|
||||
Implemented `store.go` per the plan: `Acquire` takes the whole candidate
|
||||
set so selection and insertion happen under the one store mutex (no
|
||||
overcommit between concurrent requests), selection is a linear scan +
|
||||
`slices.SortFunc` on `(activeLeases asc, latency asc, name asc)`,
|
||||
`AcquireStats{Considered, AtCapacity, InCooldown}` feeds Step 7's 409
|
||||
body, cooldowns live in a `map[{proxy, target}]time.Time` (empty target =
|
||||
global pool), and expired leases are retained for `CooldownWindow` past
|
||||
their TTL so a late `Report` — arriving exactly when a proxy is being
|
||||
rate-limited — still resolves and records its cooldown.
|
||||
|
||||
Semantics pinned against the spec (§8) rather than guessed:
|
||||
|
||||
- Report results are exactly `ok | rate_limited | banned` (`ParseResult`
|
||||
gives the API layer its 400 check). `rate_limited` and `banned` both
|
||||
record a cooldown for the same window; `ok` records nothing.
|
||||
Distinguishing ban duration from rate-limit duration would be a second
|
||||
knob the spec doesn't ask for — noted for the Decisions section.
|
||||
- Cooldown scoping: the global cooldown (empty target) always applies; a
|
||||
target-scoped cooldown additionally blocks acquisitions for that target;
|
||||
acquisitions without a target see only the global pool ("a proxy
|
||||
rate-limited by one site is still fine for everyone else").
|
||||
- A `Report` without a target falls back to the lease's own target before
|
||||
falling back to global — so a client that leased with a target doesn't
|
||||
accidentally poison the whole proxy by omitting it in the report.
|
||||
|
||||
Design notes:
|
||||
|
||||
- **Correctness never depends on the sweep.** Every read path
|
||||
(`Acquire`/`ActiveCount`/`Counts`) compares `ExpiresAt` against the
|
||||
injected clock, so TTL expiry frees capacity immediately even if the
|
||||
background loop hasn't run; the sweep is purely garbage collection. The
|
||||
plan's `ExpireLoop` became `Start(ctx)` + `NeedLeaderElection() false`
|
||||
so the store satisfies `manager.Runnable` directly — Step 10 just
|
||||
`mgr.Add(store)`s it. Not leader-elected because lease state is
|
||||
per-process and must expire wherever the discovery API is serving.
|
||||
- The store knows nothing about Proxy objects — `Candidate` carries the
|
||||
opaque key, `MaxLeases`, and latency; the discovery layer does the
|
||||
health/attribute filtering. The spec's `LeaseStore` interface will be
|
||||
defined consumer-side in `internal/discovery` (Step 7), per Go idiom;
|
||||
this package exports only the concrete in-memory `*Store`.
|
||||
- Lease IDs come from `crypto/rand.Text()` (Go 1.24+); returned `Lease`
|
||||
values are copies so callers can't mutate store internals.
|
||||
|
||||
Tests (94.8% coverage, `-race -count=2` clean): capacity + release
|
||||
freeing slots, `MaxLeases=0` unleasable, least-loaded/latency/name
|
||||
selection order, target-scoped vs global cooldown scoping, cooldown
|
||||
expiry via the injected fake clock, TTL freeing capacity with no sweep,
|
||||
report-on-expired-but-retained lease (then `ErrUnknownLease` after
|
||||
retention), `ok` recording nothing, idempotent release, `ParseResult`,
|
||||
40 concurrent acquires against `MaxLeases=5` granting exactly 5, and the
|
||||
`Start` loop sweeping then stopping cleanly on cancel.
|
||||
|
||||
```bash
|
||||
go test -race -count=2 ./internal/lease/
|
||||
make test # whole repo green, other packages' coverage unchanged
|
||||
```
|
||||
|
||||
Worth noting: `docs/architecture.md` was not extended this step — the
|
||||
lease store is HTTP-driven, not cluster-event-driven, so its diagram
|
||||
belongs with the discovery API and lands in Step 7 (banner updated to say
|
||||
so).
|
||||
|
||||
## Step 7 — Discovery API (`internal/discovery/`)
|
||||
|
||||
Implemented `server.go` (Runnable + middleware chain) and `handlers.go`
|
||||
(the four endpoints + `proxyView` wire shape) per the plan: stdlib
|
||||
`http.ServeMux` method+wildcard routing (no third-party router — see the
|
||||
plan clarification commit: this is a stdlib feature since Go 1.22, the
|
||||
project stays on the pinned Go 1.26), middleware outermost-first recover →
|
||||
request-log → `MaxBytesReader(64KiB)` → constant-time bearer auth with
|
||||
`/healthz` exempt, empty `DISCOVERY_TOKEN` serving unauthenticated with a
|
||||
loud startup warning, `NeedLeaderElection() = false` with the plan's
|
||||
runnable-ordering rationale in the doc comment, and graceful `Shutdown`
|
||||
with a 10 s grace on ctx cancel.
|
||||
|
||||
The `LeaseStore` interface landed consumer-side in this package (spec §8
|
||||
wants handlers swappable to a CRD/Redis store); `internal/lease.*Store`
|
||||
satisfies it without modification.
|
||||
|
||||
Judgment calls the plan/spec left open:
|
||||
|
||||
- **409 arithmetic:** the store only ever sees healthy candidates, so its
|
||||
`Considered` excludes unhealthy matches. The handler counts unhealthy
|
||||
selector-matches itself and reports `considered = healthy + unhealthy`,
|
||||
keeping the plan's example arithmetic (7 = 2+2+3) consistent.
|
||||
- **TTL handling:** omitted/zero `ttlSeconds` → 300 s default; negative or
|
||||
above `MaxLeaseTTL` (default 1h, flag in Step 10) → 400 `invalid_ttl`
|
||||
rather than silent clamping — a client asking for a week-long lease
|
||||
should find out, not get an hour quietly.
|
||||
- **Grant response includes the fresh `activeLeases`** (the just-granted
|
||||
lease counted), read back via `Store.Counts()` after the acquire.
|
||||
- Proxies with a deletionTimestamp are filtered out of both list and
|
||||
candidate selection — a proxy mid-teardown shouldn't be advertised.
|
||||
|
||||
Tests (87.3% coverage, `-race -count=2` clean, green on first run):
|
||||
httptest over the real handler chain with a fake cache reader and a real
|
||||
`lease.Store` — auth on/off/wrong-token/healthz-exempt, list filtering
|
||||
(attributes, healthy, combined, empty-is-200), grant shape (201, default
|
||||
TTL, lowest-latency pick, RFC3339 expiresAt, activeLeases=1), the full
|
||||
409 body arithmetic, invalid TTL/body/result, idempotent 204 release,
|
||||
report→cooldown→409 round-trip, 404 on unknown lease, and a real
|
||||
`Start` on `127.0.0.1:0` (via the new `BoundAddr()` accessor) serving
|
||||
healthz then shutting down cleanly on cancel.
|
||||
|
||||
```bash
|
||||
go test -race -count=2 ./internal/discovery/
|
||||
make test # whole repo green
|
||||
```
|
||||
|
||||
Worth noting: `go mod tidy` promoted `github.com/go-logr/logr` from
|
||||
indirect to direct (the server holds a `logr.Logger` field). The
|
||||
`--discovery-addr`, `--max-lease-ttl` flags and the `DISCOVERY_TOKEN`
|
||||
Secret mount arrive with `cmd/main.go` in Step 10. `docs/architecture.md`
|
||||
gained §7 covering the whole HTTP path and the store's sweep Runnable.
|
||||
|
||||
## Step 8 — GCP provider (`internal/provider/gcp/`)
|
||||
|
||||
Implemented per the plan: `gcp.go` (Provider + the flattened `instancesAPI`
|
||||
test seam + providerID handling + state mapping), `insert.go` (pure
|
||||
`buildInsertRequest` + config defaults), `errors.go` (HTTP-code → taxonomy
|
||||
classification). Only the four calls the spec allows — instances Insert /
|
||||
Get / Delete / AggregatedList — and `Operation.Wait` is never called:
|
||||
Create/Delete return once the operation is submitted, `409 alreadyExists`
|
||||
on Insert and `404` on Delete both count as success, which is what makes
|
||||
repeat calls after a crash correct.
|
||||
|
||||
Dependency added (the plan's environment check pinned it):
|
||||
|
||||
```bash
|
||||
go get cloud.google.com/go/compute@v1.65.0 google.golang.org/api@latest
|
||||
# resolved google.golang.org/api v0.292.0; go mod tidy pulled the auth/gax chain
|
||||
```
|
||||
|
||||
Key shapes, all straight from the plan:
|
||||
|
||||
- **providerID `zones/<zone>/instances/<name>`** — Get/Delete parse the
|
||||
zone out of the ID instead of re-reading `spec.placement.zone`, which is
|
||||
wrong exactly when a zone edit is the replacement being processed.
|
||||
- **The seam is not an SDK mirror** — verified the plan's premise against
|
||||
the vendored source before designing around it:
|
||||
`InstancesScopedListPairIterator` has an unexported `nextFunc`, so a
|
||||
fake cannot construct one. The seam flattens `AggregatedList` to
|
||||
`[]*computepb.Instance` and returns operations as just their name.
|
||||
- `AggregatedList` sets `ReturnPartialSuccess: true` (one unreachable
|
||||
zone must not fail a GC sweep) and filters by
|
||||
`labels.proxy-operator-managed = true`.
|
||||
- `RUNNING` without a `NatIP` maps to `Provisioning` — never publish an
|
||||
empty IP. Unknown/new GCP statuses map to `Stopped`: the reconciler's
|
||||
response is delete-and-recreate, always safe for cattle.
|
||||
- Classification: 404→NotFound; 429 and 403-with-
|
||||
`quotaExceeded`/`rateLimitExceeded`→Quota; 400/401/403-other→Permanent;
|
||||
everything else (408, 5xx, network, unknown)→Transient.
|
||||
|
||||
Judgment call: `placement.zone`/`machineType`/`image` are all required at
|
||||
`Create` — missing values fail as `ErrPermanent` with a message naming
|
||||
the empty fields, rather than inventing defaults the spec doesn't define.
|
||||
A wrong guess here would silently create billable VMs of an arbitrary
|
||||
shape; a Failed condition telling the user what to set is strictly better.
|
||||
|
||||
Tests (75.2%, `-race -count=2` clean, green on first run): the plan's
|
||||
primary field-by-field `buildInsertRequest` assertion (machine-type URL,
|
||||
boot disk, the exact `{External NAT, ONE_TO_ONE_NAT}` access config,
|
||||
user-data metadata, GC labels, network tag) plus config overrides and
|
||||
no-metadata-without-cloud-init; the full classification table including
|
||||
`errors.Is` AND `errors.As` through the multi-unwrap; and fake-seam tests
|
||||
for zone-qualified IDs, 409-is-success, permanent-on-bad-placement (no
|
||||
API call made), the nine-row state-mapping table, 404 paths, malformed
|
||||
providerIDs, and the ListByTag filter/partial-success assertions. The
|
||||
uncovered remainder is `New()` (dials real Google with ADC) and the
|
||||
`realInstances` adapter — the same deliberately-untested posture as the
|
||||
kubernetes provider's `New()`.
|
||||
|
||||
Worth noting: gopls suggested replacing `proto.String(x)` with Go 1.26's
|
||||
`new(x)` expression; left as `proto.String` — it is the universal
|
||||
protobuf-construction idiom and matches every example in the SDK docs.
|
||||
Registry wiring (`"gcp": gcp.New`) happens at the composition root in
|
||||
Step 10, as designed in Step 2. `docs/architecture.md` §5 now shows both
|
||||
providers' call mappings.
|
||||
|
||||
## Step 9 — Orphan GC + metrics
|
||||
|
||||
Implemented `internal/gc/gc.go` (the `Sweeper` Runnable) and
|
||||
`internal/metrics/metrics.go` (explicit-registration metric set), plus the
|
||||
`provider.WithMetrics` decorator deferred from Step 2 into
|
||||
`internal/provider/metrics.go`, and the observation hooks in the health
|
||||
engine and discovery server.
|
||||
|
||||
**GC**, per the plan: `NeedLeaderElection() = true` (destructive ⇒ single
|
||||
writer), 10 min ticker with the first sweep one full interval after start,
|
||||
per-provider `ListByTag` with log-and-continue on provider errors, and a
|
||||
kill requires all three of: our UID label present, older than `MinAge`
|
||||
(10 min), and the UID matching no existing CR — where a CR with a
|
||||
`deletionTimestamp` still counts as live (its finalizer owns that
|
||||
deletion; GC racing it would double-delete). Two safety behaviors worth
|
||||
naming: a failed Proxy `List` skips the whole sweep (an unreadable live
|
||||
set proves nothing orphaned), and the namespace-scope guard makes `Start`
|
||||
refuse to run against a namespace-restricted cache unless
|
||||
`--gc-allow-namespaced` is explicit, with the flag named in the error.
|
||||
One deviation of record: the plan says kills log "at Warn", but logr has
|
||||
no Warn level — kills log at Info with a `WARNING:` prefix carrying
|
||||
provider, providerID, and UID, same convention as the discovery server's
|
||||
empty-token warning.
|
||||
|
||||
**Metrics**, per the plan: no `init()` — `Metrics.Register(reg, phases,
|
||||
leases)` is called explicitly by the composition root (Step 10), which is
|
||||
also what lets every test use a fresh registry (asserted by a test that
|
||||
registers two sets on two registries). `proxy_operator_proxies{phase}`
|
||||
and `proxy_operator_leases_active` are scrape-time collectors fed by
|
||||
closures — a reconcile-incremented gauge drifts and leaks series on
|
||||
delete; reading the source of truth at scrape time cannot. The
|
||||
probe vectors observe **every** probe (status stays transition-only;
|
||||
metrics carry the high-frequency signal), and the health engine calls
|
||||
`ForgetProxy` when it prunes a state entry so per-proxy series don't leak.
|
||||
`provider_requests_total{provider,op,result}` comes from the
|
||||
`WithMetrics` decorator — the one place `Class()` is called purely for
|
||||
observability, with results labelled ok / not_found / quota_exceeded /
|
||||
permanent / transient.
|
||||
|
||||
**Decoupling shape:** each consuming package defines its own small
|
||||
recorder interface (`health.ProbeMetrics`, `discovery.LeaseMetrics`,
|
||||
`provider.RequestRecorder`); `metrics.Metrics` satisfies all of them
|
||||
structurally. Only `cmd/main.go` will import `internal/metrics`.
|
||||
|
||||
Tests (`-race -count=2` clean): GC's true-orphan matrix in one sweep
|
||||
(live kept, deleting-CR kept, young kept, unlabelled kept, orphan
|
||||
deleted), broken-provider isolation, list-failure skips sweep, the
|
||||
namespace guard both ways, and the Start loop sweeping then stopping;
|
||||
metrics via `prometheus/testutil` — `GatherAndCompare` on the scrape-time
|
||||
collectors, ForgetProxy dropping series, outcome/result label counts;
|
||||
the decorator's five-way classification table with error passthrough
|
||||
asserted. Coverage: gc 86.1%, metrics 95.0%, provider up to 97.1%;
|
||||
health/discovery re-ran green with the hooks in place.
|
||||
|
||||
```bash
|
||||
go test -race -count=2 ./internal/gc/ ./internal/metrics/ ./internal/provider/ ./internal/health/ ./internal/discovery/
|
||||
make test # whole repo green
|
||||
```
|
||||
|
||||
Worth noting: `prometheus/client_golang` was already in the module via
|
||||
controller-runtime's metrics server, so no new dependency — `go mod tidy`
|
||||
just promoted it to direct. `docs/architecture.md` gained §8 (GC sweep)
|
||||
and §9 (metrics shape).
|
||||
|
||||
## Step 10 — Wiring, config, docs
|
||||
|
||||
The composition root and everything around it. `cmd/main.go` now: parses
|
||||
the plan's flag set (plus `--gc-allow-namespaced` from Step 9), loads the
|
||||
provider config first and fails fast, builds the registry with
|
||||
`{"kubernetes": kubernetes.New, "gcp": gcp.New}`, wraps every provider in
|
||||
`provider.WithMetrics`, then adds the lease store, health engine,
|
||||
discovery server, and GC sweeper to one manager and hands the reconciler
|
||||
its providers + health snapshotter + events channel. Metrics register on
|
||||
controller-runtime's global registry with scrape-time closures (phase
|
||||
counts from the cache, active leases summed from `store.Counts()`).
|
||||
|
||||
Two cache decisions became concrete here:
|
||||
|
||||
- The Secret cache is restricted to Secrets labelled
|
||||
`crawl.example.com/cloud-init=true` (new constant
|
||||
`v1alpha1.LabelCloudInit`) — the operator holds cluster-wide Secret
|
||||
read RBAC, and an unrestricted cache would hold every Secret in scope.
|
||||
Consequence documented in the README: an unlabelled referenced Secret
|
||||
is invisible → `CloudInitError`.
|
||||
- `--proxy-namespace` restricts the whole cache via `DefaultNamespaces`
|
||||
and flips the GC sweeper's `NamespaceRestricted` guard.
|
||||
|
||||
Manifests: `config/manager/manager.yaml` gained the
|
||||
`--providers-config` arg, the optional `DISCOVERY_TOKEN` secretKeyRef
|
||||
(`optional: true` — without the Secret the API runs unauthenticated with
|
||||
its loud warning), the ConfigMap volume mount, and containerPort 8090;
|
||||
new `config/manager/providers_config.yaml` (kubernetes-only default) and
|
||||
`config/default/discovery_service.yaml`. RBAC: the pods marker landed in
|
||||
the controller RBAC block (cluster-scoped role — the kubernetes
|
||||
provider's ListByTag spans namespaces). The plan's events RBAC was
|
||||
deliberately omitted: nothing wires an EventRecorder, and unused verbs
|
||||
are lint noise — recorded in Decisions.
|
||||
|
||||
Samples: `proxy_kubernetes.yaml` / `proxy_gcp.yaml` (with a working
|
||||
Squid-installing cloud-init) / `proxy_external.yaml` replace the scaffold
|
||||
placeholder; `providers-config.yaml` documents both provider blocks;
|
||||
`hack/providers-dev.yaml` + a new `run-dev` Makefile target run locally
|
||||
with plain-HTTP metrics:
|
||||
|
||||
```bash
|
||||
make run-dev
|
||||
# go run ./cmd/main.go --providers-config hack/providers-dev.yaml \
|
||||
# --metrics-bind-address :8080 --metrics-secure=false
|
||||
```
|
||||
|
||||
Docs: README rewritten per the plan (60-second architecture, kind
|
||||
quickstart, in-cluster deploy incl. token Secret creation, GCP setup with
|
||||
the IAM roles, the two prominent caveats, the no-substitutions version
|
||||
pins note). `docs/architecture.md` gained the components table and the
|
||||
full Decisions section — the plan's listed decisions plus everything
|
||||
accumulated in this log (RequeueNow, quota≠Failed, 409 arithmetic,
|
||||
TTL-400-not-clamp, GCP required placement, unknown-status→Stopped,
|
||||
banned==rate_limited window, GC logging convention, the Secret label
|
||||
contract, events-RBAC omission, logr-not-slog). `CHANGELOG.md` got its
|
||||
first entry with a real timestamp.
|
||||
|
||||
Verified: `make test` green across the repo (coverage unchanged),
|
||||
`bin/kustomize build config/default` and `config/samples` render clean,
|
||||
`make build` produces the binary, e2e-tagged build + vet clean. The kind
|
||||
end-to-end run is deliberately still ahead — it is the Verification
|
||||
step's job, after Step 11 closes the remaining test gaps.
|
||||
|
||||
Worth noting: `make run-dev` passes `--metrics-secure=false` because the
|
||||
scaffold's secure-serving default requires authn/authz reachability that
|
||||
a local process doesn't have; in-cluster deployments keep the secure
|
||||
default from the kustomize patch. The `providers` map wrapping happens
|
||||
*before* any consumer sees it, so the reconciler and GC only ever hold
|
||||
instrumented providers.
|
||||
|
||||
## Step 11 — Tests
|
||||
|
||||
Most of the plan's Step 11 list was deliberately front-loaded into the
|
||||
step that built each component (the action-table suite, computePhase and
|
||||
SpecHash tables, lease-store matrix incl. the concurrent `-race` case,
|
||||
discovery httptest suite, health threshold/CONNECT tests, GCP
|
||||
`buildInsertRequest` + classification, name-derivation tests from
|
||||
Step 2). This step closed what remained — the envtest-only coverage —
|
||||
and audited the list item by item.
|
||||
|
||||
Added to `internal/controller/proxy_controller_test.go`:
|
||||
|
||||
- **The CEL cases only a real API server can test** (fake clients run
|
||||
neither CEL nor structural defaulting): six invalid-create rejections
|
||||
(Managed-without-provider, External-with-provider,
|
||||
External-without-endpoint, Managed-with-endpoint, cloudInit
|
||||
both/neither), mode-mutation rejection, provider mutation *and removal*
|
||||
rejection (the `has(self.x)==has(oldSelf.x)` form exists exactly for
|
||||
the removal case), and the `+kubebuilder:default={}` assertion — a
|
||||
Proxy created with no `healthCheck` comes back with every nested
|
||||
default materialized, plus port and maxLeases defaults.
|
||||
- **Ready-through-health**: Managed proxy walks to Running (phase still
|
||||
Provisioning — "no health verdict yet must not be Ready"), then a fake
|
||||
`HealthSnapshotter` supplies a healthy snapshot and the phase flips to
|
||||
Ready with latency in status.
|
||||
- **Quota + permanent, envtest edition**: quota → condition
|
||||
QuotaExceeded, `RequeueAfter = QuotaRetry`, nil error, phase *not*
|
||||
Failed; then permanent → Failed and the generation latch provably stops
|
||||
further provider calls.
|
||||
- **Adopt**: strip the spec-hash annotation off a Running proxy (as an
|
||||
operator upgrade with a changed hash-input struct would), reconcile,
|
||||
and assert the annotation is restored byte-identical, the providerID
|
||||
unchanged, and zero provider deletes.
|
||||
|
||||
One repo-wide change: `make test` now runs with `-race` (the plan's
|
||||
"everything runs with -race" was previously only true of the manual
|
||||
verification runs, not the canonical target):
|
||||
|
||||
```make
|
||||
go test -race $$(go list ./... | grep -v /e2e) -coverprofile cover.out
|
||||
```
|
||||
|
||||
Everything green on the first run of the new specs; full suite ~9 s for
|
||||
the controller package under race, `-short` still skips envtest in 0.6 s.
|
||||
|
||||
Worth noting: the provider-removal CEL test has a subtlety worth keeping —
|
||||
removing `provider` alone would also trip the required-iff rule, so the
|
||||
test flips mode and adds an endpoint in the same update to isolate the
|
||||
immutability rules as the thing that rejects. The plan's remaining
|
||||
checklist item is Verification: the throwaway-kind-cluster run of the
|
||||
README quickstart, then push + MR.
|
||||
|
||||
## Verification — kind end-to-end
|
||||
|
||||
Static checks first (`go vet ./...`, `make build`, full `make test` with
|
||||
`-race`): all green. Then the real thing, per the spec's §13 "run the kind
|
||||
quickstart yourself and fix what breaks" — and two things broke, both now
|
||||
fixed.
|
||||
|
||||
**Finding 1 — `make run-dev` cannot produce a Ready proxy on kind.** The
|
||||
operator on the host provisions the pod fine (Provisioned=True, IP
|
||||
published), but the health probe originates on the host, and kind pod IPs
|
||||
(10.244.x.x) are not host-routable — every probe fails by construction
|
||||
and the proxy latches `Unhealthy`:
|
||||
|
||||
```text
|
||||
Healthy=False: Get "https://www.gstatic.com/generate_204":
|
||||
proxyconnect tcp: dial tcp 10.244.0.5:3128: connect: connection refused
|
||||
```
|
||||
|
||||
Everything around the failure worked exactly as designed (thresholds,
|
||||
condition, phase, and the finalizer delete ran clean from the host). Fix:
|
||||
the README quickstart now deploys the operator **in-cluster**
|
||||
(docker-build → kind load → deploy → port-forward 8090), with the
|
||||
run-dev limitation documented in both the quickstart and the Development
|
||||
section.
|
||||
|
||||
**Finding 2 — Squid was OOM-killed at startup in-cluster.** With the
|
||||
operator deployed in-cluster the pod crash-looped (`OOMKilled`, empty
|
||||
logs). Root cause: squid sizes its file-descriptor tables from
|
||||
`RLIMIT_NOFILE`, and containerd under kind sets that effectively
|
||||
unlimited (~10^9) — squid allocates gigabytes before it ever listens.
|
||||
Fix in the generated config (`internal/provider/kubernetes/pod.go`):
|
||||
`max_filedescriptors 1024` (the load-bearing line) plus `cache_mem 16 MB`
|
||||
(a crawling forward proxy gains nothing from squid's 256 MB default),
|
||||
with a regression assertion added to `pod_test.go`.
|
||||
|
||||
**With both fixes, the full pass:**
|
||||
|
||||
```bash
|
||||
kind create cluster --name proxy-operator-demo
|
||||
make install
|
||||
make docker-build IMG=egress-proxies-operator:dev
|
||||
kind load docker-image egress-proxies-operator:dev --name proxy-operator-demo
|
||||
make deploy IMG=egress-proxies-operator:dev
|
||||
kubectl apply -f config/samples/proxy_kubernetes.yaml
|
||||
# → Ready 10.244.0.9 lat=89ms Provisioned=True Healthy=True (~30 s)
|
||||
kubectl -n egress-proxies-operator-system port-forward svc/...-discovery-service 8090:8090 &
|
||||
curl -s 'localhost:8090/v1/proxies?healthy=true' # count:1, latencyMillis:89
|
||||
curl -s -XPOST localhost:8090/v1/leases -d '{"selector":{"geo":"local"},"ttlSeconds":300}'
|
||||
# → 201 {leaseID, proxy(activeLeases:1), expiresAt, ttlSeconds:300}
|
||||
curl -XPOST .../report -d '{"result":"rate_limited","target":"example.com"}' # 204
|
||||
curl -XPOST /v1/leases -d '{...,"target":"example.com"}' # 409 {inCooldown:1} ✓
|
||||
curl -XDELETE /v1/leases/<id> # 204, and 204 again ✓
|
||||
kubectl delete -f config/samples/proxy_kubernetes.yaml # finalizer: pod Terminating, CR gone
|
||||
kind delete cluster --name proxy-operator-demo
|
||||
```
|
||||
|
||||
A real Squid pod went Ready through a real CONNECT probe, a lease was
|
||||
held on it, the cooldown machinery answered a 409 with correct
|
||||
arithmetic, and the finalizer cleaned up — the plan's success bar, met
|
||||
with the actual product.
|
||||
|
||||
Worth noting: `make deploy` runs `kustomize edit set image` and mutates
|
||||
`config/manager/kustomization.yaml` in the working tree — reverted before
|
||||
committing (the repo keeps the pinned stanza). The health probe's ~89 ms
|
||||
latency is gstatic-through-squid from a kind pod on this machine;
|
||||
metrics-side observations were not separately checked in-cluster (covered
|
||||
by unit tests).
|
||||
|
||||
@@ -391,7 +391,10 @@ pods would refuse connections while still being Service endpoints. The 1-replica
|
||||
constraint comes from lease state being per-process, which the spec already accepts —
|
||||
both facts go in the README caveats.
|
||||
|
||||
stdlib `http.ServeMux` with Go 1.22 method+wildcard patterns:
|
||||
stdlib `http.ServeMux` using its method+wildcard patterns (`"GET /path"`,
|
||||
`"/{id}"` + `r.PathValue`) — a stdlib feature available since Go 1.22, used
|
||||
here so no third-party router is needed; the project itself stays on the
|
||||
pinned Go 1.26:
|
||||
`GET /v1/proxies`, `POST /v1/leases`, `DELETE /v1/leases/{id}`,
|
||||
`POST /v1/leases/{id}/report`, plus unauthenticated `GET /healthz`.
|
||||
Middleware outermost-first: recover → request-log → `MaxBytesReader(64KiB)` → bearer
|
||||
|
||||
53
go.mod
53
go.mod
@@ -3,8 +3,13 @@ module gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute v1.65.0
|
||||
github.com/go-logr/logr v1.4.3
|
||||
github.com/onsi/ginkgo/v2 v2.27.4
|
||||
github.com/onsi/gomega v1.39.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
google.golang.org/api v0.292.0
|
||||
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
|
||||
k8s.io/api v0.36.0
|
||||
k8s.io/apimachinery v0.36.0
|
||||
k8s.io/client-go v0.36.0
|
||||
@@ -13,7 +18,10 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cel.dev/expr v0.25.2 // indirect
|
||||
cloud.google.com/go/auth v0.22.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
github.com/Masterminds/semver/v3 v3.4.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@@ -26,7 +34,6 @@ require (
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-logr/zapr v1.3.0 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.21.0 // indirect
|
||||
@@ -37,17 +44,20 @@ require (
|
||||
github.com/google/gnostic-models v0.7.0 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.23.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_golang v1.23.2 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/procfs v0.19.2 // indirect
|
||||
@@ -56,33 +66,34 @@ require (
|
||||
github.com/stoewer/go-strcase v1.3.0 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.54.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
|
||||
golang.org/x/mod v0.32.0 // indirect
|
||||
golang.org/x/net v0.49.0 // indirect
|
||||
golang.org/x/oauth2 v0.34.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.40.0 // indirect
|
||||
golang.org/x/term v0.39.0 // indirect
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
golang.org/x/tools v0.41.0 // indirect
|
||||
golang.org/x/mod v0.37.0 // indirect
|
||||
golang.org/x/net v0.57.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/term v0.45.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/tools v0.47.0 // indirect
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/grpc v1.79.3 // indirect
|
||||
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
|
||||
google.golang.org/grpc v1.83.0 // indirect
|
||||
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
||||
102
go.sum
102
go.sum
@@ -1,5 +1,15 @@
|
||||
cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
|
||||
cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
|
||||
cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs=
|
||||
cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
|
||||
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
|
||||
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
|
||||
cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ=
|
||||
cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/compute v1.65.0 h1:K0a3NRvazE7sZn5qswwI6BtlaZv1fgR5wFop5LZCLz8=
|
||||
cloud.google.com/go/compute v1.65.0/go.mod h1:vFq+Ztj9Rzhc8zf1t6hGp/6NdrEVG1GakkyVRQPRgKc=
|
||||
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
|
||||
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
|
||||
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
|
||||
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
|
||||
@@ -68,8 +78,14 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
|
||||
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
|
||||
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.19 h1:mMOE7DN2+p76/EdIrmAy9B9bH+yC4563vmnJ34QR8i4=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k=
|
||||
github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE=
|
||||
github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
@@ -154,22 +170,22 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0=
|
||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
|
||||
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
|
||||
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
||||
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
|
||||
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
@@ -182,36 +198,42 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
|
||||
golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
|
||||
golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
|
||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
||||
golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
|
||||
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
|
||||
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
|
||||
golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
|
||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
|
||||
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
|
||||
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
||||
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
|
||||
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/api v0.292.0 h1:Ewiwo/GTtiaPZSNAZQUcWLh8AYDEoPmIXyJfeoTSMHU=
|
||||
google.golang.org/api v0.292.0/go.mod h1:07kjmMnFGm2RQuCza2EZM/5N68G/fVvFb1xKjWqoFA0=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
|
||||
google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
|
||||
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
|
||||
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
6
hack/providers-dev.yaml
Normal file
6
hack/providers-dev.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
# Providers config for `make run-dev`: local development against the
|
||||
# current kubeconfig context (e.g. a kind cluster). Only the
|
||||
# kubernetes-pod provider — no cloud credentials needed.
|
||||
providers:
|
||||
- name: kubernetes
|
||||
type: kubernetes
|
||||
@@ -82,6 +82,9 @@ type ProxyReconciler struct {
|
||||
// +kubebuilder:rbac:groups=crawl.example.com,resources=proxies/status,verbs=get;update;patch
|
||||
// +kubebuilder:rbac:groups=crawl.example.com,resources=proxies/finalizers,verbs=update
|
||||
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
|
||||
// The pod verbs are for the kubernetes-pod provider; cluster-scoped, since
|
||||
// its ListByTag enumerates the operator's Pods across all namespaces.
|
||||
// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;delete
|
||||
|
||||
// Reconcile fetches the Proxy named by req into p (r.Get fills the struct
|
||||
// through the pointer), dispatches to the delete/external/managed state
|
||||
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
@@ -233,6 +234,105 @@ var _ = Describe("Proxy controller", func() {
|
||||
Expect(apierrors.IsNotFound(err)).To(BeTrue(), "proxy should be fully deleted")
|
||||
})
|
||||
|
||||
It("reaches Ready once the health engine has a verdict", func() {
|
||||
const name = "e2e-ready"
|
||||
stub := &stubProvider{createID: "inst-rdy"}
|
||||
r := newEnvtestReconciler(stub)
|
||||
DeferCleanup(func() { cleanup(r, stub, name) })
|
||||
|
||||
Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
|
||||
Spec: managedSpec(),
|
||||
})).To(Succeed())
|
||||
_, err := envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
_, err = envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
stub.getInst = &provider.Instance{ID: "inst-rdy", IP: "10.3.3.3", State: provider.StateRunning}
|
||||
_, err = envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseProvisioning),
|
||||
"no health verdict yet — must not be Ready")
|
||||
|
||||
By("supplying a healthy snapshot")
|
||||
r.Health = fakeSnapshotter{ok: true, snap: health.Snapshot{
|
||||
Healthy: true, Latency: 21 * time.Millisecond, LastProbe: time.Now(),
|
||||
}}
|
||||
_, err = envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
p := fetch(name)
|
||||
Expect(p.Status.Phase).To(Equal(crawlv1alpha1.PhaseReady))
|
||||
Expect(p.Status.LatencyMillis).To(Equal(int64(21)))
|
||||
})
|
||||
|
||||
It("treats quota exhaustion as a wait and a permanent error as Failed", func() {
|
||||
const name = "e2e-errors"
|
||||
stub := &stubProvider{
|
||||
createErr: provider.Wrap(provider.ErrQuotaExceeded, "create", "stub", "", nil),
|
||||
}
|
||||
r := newEnvtestReconciler(stub)
|
||||
DeferCleanup(func() { cleanup(r, stub, name) })
|
||||
|
||||
Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
|
||||
Spec: managedSpec(),
|
||||
})).To(Succeed())
|
||||
_, err := envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("quota: condition set, slow requeue, phase NOT Failed")
|
||||
res, err := envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred(), "quota must not count as an error (stays off the backoff curve)")
|
||||
Expect(res.RequeueAfter).To(Equal(r.QuotaRetry))
|
||||
p := fetch(name)
|
||||
cond := apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionProvisioned)
|
||||
Expect(cond.Reason).To(Equal(ReasonQuotaExceeded))
|
||||
Expect(p.Status.Phase).NotTo(Equal(crawlv1alpha1.PhaseFailed))
|
||||
|
||||
By("permanent: phase Failed and no further provider calls")
|
||||
stub.createErr = provider.Wrap(provider.ErrPermanent, "create", "stub", "", nil)
|
||||
_, err = envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseFailed))
|
||||
callsAfterLatch := stub.createCalls
|
||||
_, err = envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(stub.createCalls).To(Equal(callsAfterLatch), "the latch must stop provider calls")
|
||||
})
|
||||
|
||||
It("adopts an instance when the spec-hash annotation is stripped", func() {
|
||||
const name = "e2e-adopt"
|
||||
stub := &stubProvider{createID: "inst-adopt"}
|
||||
r := newEnvtestReconciler(stub)
|
||||
DeferCleanup(func() { cleanup(r, stub, name) })
|
||||
|
||||
Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
|
||||
Spec: managedSpec(),
|
||||
})).To(Succeed())
|
||||
_, err := envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
_, err = envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
stub.getInst = &provider.Instance{ID: "inst-adopt", IP: "10.4.4.4", State: provider.StateRunning}
|
||||
_, err = envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
originalHash := fetch(name).Annotations[crawlv1alpha1.AnnotationSpecHash]
|
||||
Expect(originalHash).NotTo(BeEmpty())
|
||||
|
||||
By("stripping the annotation, as an operator-version upgrade with a changed hash input would")
|
||||
p := fetch(name)
|
||||
delete(p.Annotations, crawlv1alpha1.AnnotationSpecHash)
|
||||
Expect(k8sClient.Update(ctx, p)).To(Succeed())
|
||||
|
||||
_, err = envReconcile(r, name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
p = fetch(name)
|
||||
Expect(p.Annotations[crawlv1alpha1.AnnotationSpecHash]).To(Equal(originalHash), "hash must be restored")
|
||||
Expect(p.Status.ProviderID).To(Equal("inst-adopt"), "adoption must keep the instance")
|
||||
Expect(stub.deleteCalls).To(BeZero(), "adoption must never replace")
|
||||
})
|
||||
|
||||
It("tracks an External proxy without touching providers", func() {
|
||||
const name = "e2e-external"
|
||||
stub := &stubProvider{}
|
||||
@@ -264,3 +364,157 @@ var _ = Describe("Proxy controller", func() {
|
||||
Expect(apierrors.IsNotFound(err)).To(BeTrue())
|
||||
})
|
||||
})
|
||||
|
||||
// These specs assert the CRD's CEL rules and structural defaulting against
|
||||
// the real envtest API server — the fake client runs neither, which is the
|
||||
// documented caveat on the action-table unit tests.
|
||||
var _ = Describe("Proxy CRD validation (CEL)", func() {
|
||||
const ns = "default"
|
||||
|
||||
managed := func(name string) *crawlv1alpha1.Proxy {
|
||||
return &crawlv1alpha1.Proxy{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
|
||||
Spec: crawlv1alpha1.ProxySpec{
|
||||
Mode: crawlv1alpha1.ModeManaged,
|
||||
Provider: "stub",
|
||||
},
|
||||
}
|
||||
}
|
||||
external := func(name string) *crawlv1alpha1.Proxy {
|
||||
return &crawlv1alpha1.Proxy{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
|
||||
Spec: crawlv1alpha1.ProxySpec{
|
||||
Mode: crawlv1alpha1.ModeExternal,
|
||||
Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
|
||||
},
|
||||
}
|
||||
}
|
||||
mustCreate := func(p *crawlv1alpha1.Proxy) {
|
||||
GinkgoHelper()
|
||||
Expect(k8sClient.Create(ctx, p)).To(Succeed())
|
||||
DeferCleanup(func() { _ = k8sClient.Delete(ctx, p) })
|
||||
}
|
||||
|
||||
It("rejects invalid creates", func() {
|
||||
invalid := []struct {
|
||||
about string
|
||||
spec crawlv1alpha1.ProxySpec
|
||||
want string
|
||||
}{
|
||||
{
|
||||
about: "Managed without provider",
|
||||
spec: crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged},
|
||||
want: "provider is required when mode is Managed",
|
||||
},
|
||||
{
|
||||
about: "External with provider",
|
||||
spec: crawlv1alpha1.ProxySpec{
|
||||
Mode: crawlv1alpha1.ModeExternal, Provider: "stub",
|
||||
Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
|
||||
},
|
||||
want: "provider must not be set when mode is External",
|
||||
},
|
||||
{
|
||||
about: "External without endpoint",
|
||||
spec: crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeExternal},
|
||||
want: "endpoint is required when mode is External",
|
||||
},
|
||||
{
|
||||
about: "Managed with endpoint",
|
||||
spec: crawlv1alpha1.ProxySpec{
|
||||
Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
|
||||
Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
|
||||
},
|
||||
want: "endpoint must not be set when mode is Managed",
|
||||
},
|
||||
{
|
||||
about: "cloudInit with both inline and secretRef",
|
||||
spec: crawlv1alpha1.ProxySpec{
|
||||
Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
|
||||
CloudInit: &crawlv1alpha1.CloudInitSpec{
|
||||
Inline: "#cloud-config",
|
||||
SecretRef: &crawlv1alpha1.SecretKeySelector{Name: "s"},
|
||||
},
|
||||
},
|
||||
want: "exactly one of inline or secretRef",
|
||||
},
|
||||
{
|
||||
about: "cloudInit with neither inline nor secretRef",
|
||||
spec: crawlv1alpha1.ProxySpec{
|
||||
Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
|
||||
CloudInit: &crawlv1alpha1.CloudInitSpec{},
|
||||
},
|
||||
want: "exactly one of inline or secretRef",
|
||||
},
|
||||
}
|
||||
for _, tc := range invalid {
|
||||
p := &crawlv1alpha1.Proxy{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "cel-invalid", Namespace: ns},
|
||||
Spec: tc.spec,
|
||||
}
|
||||
err := k8sClient.Create(ctx, p)
|
||||
Expect(err).To(HaveOccurred(), tc.about)
|
||||
Expect(err.Error()).To(ContainSubstring(tc.want), tc.about)
|
||||
}
|
||||
})
|
||||
|
||||
It("rejects mode mutation", func() {
|
||||
p := external("cel-mode-immutable")
|
||||
mustCreate(p)
|
||||
p.Spec = crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged, Provider: "stub"}
|
||||
err := k8sClient.Update(ctx, p)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("mode is immutable"))
|
||||
})
|
||||
|
||||
It("rejects provider mutation and removal", func() {
|
||||
p := managed("cel-provider-immutable")
|
||||
mustCreate(p)
|
||||
|
||||
p.Spec.Provider = "other"
|
||||
err := k8sClient.Update(ctx, p)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("provider is immutable"))
|
||||
|
||||
// Removal must also be rejected — the has()==has() form exists
|
||||
// exactly because a field-level rule would not fire on absence.
|
||||
// (Dropping provider alone would also trip the required-iff rule,
|
||||
// so flip mode too and check the immutability rules win.)
|
||||
fresh := fetchProxy(ns, "cel-provider-immutable")
|
||||
fresh.Spec.Provider = ""
|
||||
fresh.Spec.Endpoint = &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"}
|
||||
fresh.Spec.Mode = crawlv1alpha1.ModeExternal
|
||||
err = k8sClient.Update(ctx, fresh)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("immutable"))
|
||||
})
|
||||
|
||||
It("materializes every nested healthCheck default when healthCheck is omitted", func() {
|
||||
p := managed("cel-defaults")
|
||||
mustCreate(p)
|
||||
|
||||
got := fetchProxy(ns, "cel-defaults")
|
||||
// The +kubebuilder:default={} assertion: structural defaulting only
|
||||
// descends into values that exist, so without it a nil healthCheck
|
||||
// would get none of these.
|
||||
hc := got.Spec.HealthCheck
|
||||
Expect(hc).NotTo(BeNil())
|
||||
Expect(hc.ProbeURL).To(Equal(crawlv1alpha1.DefaultProbeURL))
|
||||
Expect(hc.IntervalSeconds).To(Equal(crawlv1alpha1.DefaultHealthCheckIntervalSeconds))
|
||||
Expect(hc.TimeoutSeconds).To(Equal(crawlv1alpha1.DefaultHealthCheckTimeoutSeconds))
|
||||
Expect(hc.FailureThreshold).To(Equal(crawlv1alpha1.DefaultFailureThreshold))
|
||||
Expect(hc.SuccessThreshold).To(Equal(crawlv1alpha1.DefaultSuccessThreshold))
|
||||
Expect(hc.ExpectedStatusCodes).To(Equal(crawlv1alpha1.DefaultExpectedStatusCodes))
|
||||
|
||||
Expect(got.Spec.Port).To(Equal(crawlv1alpha1.DefaultPort))
|
||||
Expect(got.Spec.MaxLeases).NotTo(BeNil())
|
||||
Expect(*got.Spec.MaxLeases).To(Equal(crawlv1alpha1.DefaultMaxLeases))
|
||||
})
|
||||
})
|
||||
|
||||
func fetchProxy(ns, name string) *crawlv1alpha1.Proxy {
|
||||
GinkgoHelper()
|
||||
p := &crawlv1alpha1.Proxy{}
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, p)).To(Succeed())
|
||||
return p
|
||||
}
|
||||
|
||||
241
internal/discovery/handlers.go
Normal file
241
internal/discovery/handlers.go
Normal file
@@ -0,0 +1,241 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
|
||||
)
|
||||
|
||||
// proxyView is the wire shape of one proxy in list and lease responses.
|
||||
type proxyView struct {
|
||||
ID string `json:"id"` // namespace/name
|
||||
IP string `json:"ip"`
|
||||
Port int32 `json:"port"`
|
||||
Attributes map[string]string `json:"attributes,omitempty"`
|
||||
Phase string `json:"phase"`
|
||||
Healthy bool `json:"healthy"`
|
||||
LatencyMillis int64 `json:"latencyMillis"`
|
||||
ActiveLeases int `json:"activeLeases"`
|
||||
MaxLeases int32 `json:"maxLeases"`
|
||||
}
|
||||
|
||||
func viewOf(p *crawlv1alpha1.Proxy, activeLeases int) proxyView {
|
||||
return proxyView{
|
||||
ID: client.ObjectKeyFromObject(p).String(),
|
||||
IP: p.EffectiveHost(),
|
||||
Port: p.EffectivePort(),
|
||||
Attributes: p.Spec.Attributes,
|
||||
Phase: string(p.Status.Phase),
|
||||
Healthy: isHealthy(p),
|
||||
LatencyMillis: p.Status.LatencyMillis,
|
||||
ActiveLeases: activeLeases,
|
||||
MaxLeases: p.MaxLeasesOrDefault(),
|
||||
}
|
||||
}
|
||||
|
||||
func isHealthy(p *crawlv1alpha1.Proxy) bool {
|
||||
return apimeta.IsStatusConditionTrue(p.Status.Conditions, crawlv1alpha1.ConditionHealthy)
|
||||
}
|
||||
|
||||
// matchesAttributes is spec.attributes equality: every selector pair must
|
||||
// be present verbatim.
|
||||
func matchesAttributes(p *crawlv1alpha1.Proxy, selector map[string]string) bool {
|
||||
for k, v := range selector {
|
||||
if p.Spec.Attributes[k] != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// GET /v1/proxies?attr.<key>=<value>&healthy=true|false
|
||||
func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
|
||||
selector := map[string]string{}
|
||||
var healthyFilter *bool
|
||||
for key, values := range r.URL.Query() {
|
||||
switch {
|
||||
case key == "healthy":
|
||||
switch values[0] {
|
||||
case "true":
|
||||
healthyFilter = ptr(true)
|
||||
case "false":
|
||||
healthyFilter = ptr(false)
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, "invalid_query",
|
||||
fmt.Sprintf("healthy must be true or false, got %q", values[0]))
|
||||
return
|
||||
}
|
||||
case strings.HasPrefix(key, "attr."):
|
||||
selector[strings.TrimPrefix(key, "attr.")] = values[0]
|
||||
}
|
||||
}
|
||||
|
||||
var list crawlv1alpha1.ProxyList
|
||||
if err := s.Reader.List(r.Context(), &list); err != nil {
|
||||
s.log.Error(err, "listing proxies")
|
||||
writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
|
||||
return
|
||||
}
|
||||
counts := s.Store.Counts()
|
||||
|
||||
views := []proxyView{}
|
||||
for i := range list.Items {
|
||||
p := &list.Items[i]
|
||||
if !p.DeletionTimestamp.IsZero() || !matchesAttributes(p, selector) {
|
||||
continue
|
||||
}
|
||||
v := viewOf(p, counts[client.ObjectKeyFromObject(p).String()])
|
||||
if healthyFilter != nil && v.Healthy != *healthyFilter {
|
||||
continue
|
||||
}
|
||||
views = append(views, v)
|
||||
}
|
||||
slices.SortFunc(views, func(a, b proxyView) int { return strings.Compare(a.ID, b.ID) })
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]any{"proxies": views, "count": len(views)})
|
||||
}
|
||||
|
||||
type leaseRequest struct {
|
||||
Selector map[string]string `json:"selector"`
|
||||
TTLSeconds int64 `json:"ttlSeconds"`
|
||||
Target string `json:"target"`
|
||||
}
|
||||
|
||||
type leaseResponse struct {
|
||||
LeaseID string `json:"leaseID"`
|
||||
Proxy proxyView `json:"proxy"`
|
||||
ExpiresAt time.Time `json:"expiresAt"`
|
||||
TTLSeconds int64 `json:"ttlSeconds"`
|
||||
}
|
||||
|
||||
// POST /v1/leases
|
||||
func (s *Server) handleAcquireLease(w http.ResponseWriter, r *http.Request) {
|
||||
var req leaseRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_body", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
ttl := time.Duration(req.TTLSeconds) * time.Second
|
||||
if req.TTLSeconds == 0 {
|
||||
ttl = defaultTTL
|
||||
}
|
||||
if ttl < 0 || ttl > s.MaxLeaseTTL {
|
||||
writeError(w, http.StatusBadRequest, "invalid_ttl",
|
||||
fmt.Sprintf("ttlSeconds must be between 1 and %d", int64(s.MaxLeaseTTL.Seconds())))
|
||||
return
|
||||
}
|
||||
|
||||
var list crawlv1alpha1.ProxyList
|
||||
if err := s.Reader.List(r.Context(), &list); err != nil {
|
||||
s.log.Error(err, "listing proxies for lease")
|
||||
writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
|
||||
return
|
||||
}
|
||||
|
||||
// The store gets only healthy, live candidates; unhealthy matches are
|
||||
// counted here because the store never sees them.
|
||||
var candidates []lease.Candidate
|
||||
byKey := map[string]*crawlv1alpha1.Proxy{}
|
||||
unhealthy := 0
|
||||
for i := range list.Items {
|
||||
p := &list.Items[i]
|
||||
if !p.DeletionTimestamp.IsZero() || !matchesAttributes(p, req.Selector) {
|
||||
continue
|
||||
}
|
||||
if !isHealthy(p) {
|
||||
unhealthy++
|
||||
continue
|
||||
}
|
||||
key := client.ObjectKeyFromObject(p).String()
|
||||
byKey[key] = p
|
||||
candidates = append(candidates, lease.Candidate{
|
||||
Proxy: key,
|
||||
MaxLeases: p.MaxLeasesOrDefault(),
|
||||
Latency: time.Duration(p.Status.LatencyMillis) * time.Millisecond,
|
||||
})
|
||||
}
|
||||
|
||||
granted, stats, err := s.Store.Acquire(r.Context(), lease.AcquireRequest{
|
||||
Candidates: candidates,
|
||||
Target: req.Target,
|
||||
TTL: ttl,
|
||||
})
|
||||
if errors.Is(err, lease.ErrNoMatch) {
|
||||
if s.Metrics != nil {
|
||||
s.Metrics.LeaseRequest("no_match")
|
||||
}
|
||||
writeJSON(w, http.StatusConflict, map[string]any{
|
||||
"error": "no_match",
|
||||
"message": "no healthy proxy with free capacity matched the selector",
|
||||
"considered": stats.Considered + unhealthy,
|
||||
"atCapacity": stats.AtCapacity,
|
||||
"inCooldown": stats.InCooldown,
|
||||
"unhealthy": unhealthy,
|
||||
})
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if s.Metrics != nil {
|
||||
s.Metrics.LeaseRequest("granted")
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, leaseResponse{
|
||||
LeaseID: granted.ID,
|
||||
Proxy: viewOf(byKey[granted.Proxy], s.Store.Counts()[granted.Proxy]),
|
||||
ExpiresAt: granted.ExpiresAt,
|
||||
TTLSeconds: int64(ttl.Seconds()),
|
||||
})
|
||||
}
|
||||
|
||||
// DELETE /v1/leases/{id} — early release, always 204: releasing an unknown
|
||||
// or already expired lease is not an error.
|
||||
func (s *Server) handleReleaseLease(w http.ResponseWriter, r *http.Request) {
|
||||
s.Store.Release(r.Context(), r.PathValue("id"))
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
type reportRequest struct {
|
||||
Result string `json:"result"`
|
||||
Target string `json:"target"`
|
||||
}
|
||||
|
||||
// POST /v1/leases/{id}/report
|
||||
func (s *Server) handleReportLease(w http.ResponseWriter, r *http.Request) {
|
||||
var req reportRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_body", err.Error())
|
||||
return
|
||||
}
|
||||
result, ok := lease.ParseResult(req.Result)
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadRequest, "invalid_result",
|
||||
fmt.Sprintf("result must be one of ok, rate_limited, banned; got %q", req.Result))
|
||||
return
|
||||
}
|
||||
if err := s.Store.Report(r.Context(), r.PathValue("id"), result, req.Target); err != nil {
|
||||
if errors.Is(err, lease.ErrUnknownLease) {
|
||||
writeError(w, http.StatusNotFound, "unknown_lease", "no such lease")
|
||||
return
|
||||
}
|
||||
s.log.Error(err, "reporting lease", "leaseID", r.PathValue("id"))
|
||||
writeError(w, http.StatusInternalServerError, "internal", "report failed")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func ptr[T any](v T) *T { return &v }
|
||||
234
internal/discovery/server.go
Normal file
234
internal/discovery/server.go
Normal file
@@ -0,0 +1,234 @@
|
||||
// Package discovery implements the HTTP API crawler clients use to find
|
||||
// and lease proxies: list healthy proxies filtered by attributes, acquire a
|
||||
// TTL-based lease, release it early, and report how a target treated the
|
||||
// proxy. Reads go through the manager's informer cache; lease state lives
|
||||
// in the injected store.
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
|
||||
)
|
||||
|
||||
// LeaseStore is what the handlers need from a lease backend. Defined here,
|
||||
// consumer-side, so a CRD- or Redis-backed store can replace the in-memory
|
||||
// one (which internal/lease's *Store satisfies) without touching handlers.
|
||||
type LeaseStore interface {
|
||||
Acquire(ctx context.Context, req lease.AcquireRequest) (*lease.Lease, lease.AcquireStats, error)
|
||||
Release(ctx context.Context, id string)
|
||||
Report(ctx context.Context, id string, result lease.Result, target string) error
|
||||
Counts() map[string]int
|
||||
}
|
||||
|
||||
// LeaseMetrics counts lease acquisitions by outcome. Implemented by
|
||||
// internal/metrics; defined here so this package carries no metrics
|
||||
// dependency.
|
||||
type LeaseMetrics interface {
|
||||
LeaseRequest(outcome string)
|
||||
}
|
||||
|
||||
const (
|
||||
defaultAddr = ":8090"
|
||||
defaultTTL = 5 * time.Minute
|
||||
defaultMaxTTL = time.Hour
|
||||
maxBodyBytes = 64 << 10
|
||||
shutdownGrace = 10 * time.Second
|
||||
readHeadTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
// Server serves the discovery API as a manager Runnable.
|
||||
type Server struct {
|
||||
// Reader lists Proxies from the manager's cache.
|
||||
Reader client.Reader
|
||||
// Store is the lease backend.
|
||||
Store LeaseStore
|
||||
// Addr is the listen address (default ":8090"; --discovery-addr).
|
||||
Addr string
|
||||
// Token is the static bearer token from DISCOVERY_TOKEN. Empty
|
||||
// disables auth — allowed for the prototype, but loudly warned about
|
||||
// at startup, because in-cluster that is a silent security hole.
|
||||
Token string
|
||||
// MaxLeaseTTL caps requested lease TTLs (default 1h; --max-lease-ttl).
|
||||
MaxLeaseTTL time.Duration
|
||||
// Metrics, when non-nil, counts lease requests by outcome.
|
||||
Metrics LeaseMetrics
|
||||
|
||||
log logr.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
boundAddr string
|
||||
}
|
||||
|
||||
// NeedLeaderElection is false, and the deployment ships replicas: 1.
|
||||
// Verified against controller-runtime's runnable ordering: caches start and
|
||||
// sync before non-leader-election runnables, so cache reads here are safe.
|
||||
// If this were leader-elected, non-leader replicas would refuse connections
|
||||
// while still being Service endpoints. The 1-replica constraint comes from
|
||||
// lease state being per-process — both facts are README caveats.
|
||||
func (s *Server) NeedLeaderElection() bool { return false }
|
||||
|
||||
// BoundAddr returns the actual listen address once Start has bound it —
|
||||
// meaningful when Addr uses port 0 (tests).
|
||||
func (s *Server) BoundAddr() string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.boundAddr
|
||||
}
|
||||
|
||||
// Start listens and serves until ctx ends, then shuts down gracefully with
|
||||
// a 10-second grace period.
|
||||
func (s *Server) Start(ctx context.Context) error {
|
||||
if s.Addr == "" {
|
||||
s.Addr = defaultAddr
|
||||
}
|
||||
if s.MaxLeaseTTL == 0 {
|
||||
s.MaxLeaseTTL = defaultMaxTTL
|
||||
}
|
||||
s.log = logf.FromContext(ctx).WithName("discovery")
|
||||
if s.Token == "" {
|
||||
s.log.Info("WARNING: DISCOVERY_TOKEN is empty — the discovery API is served without authentication")
|
||||
}
|
||||
|
||||
ln, err := net.Listen("tcp", s.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.boundAddr = ln.Addr().String()
|
||||
s.mu.Unlock()
|
||||
|
||||
srv := &http.Server{
|
||||
Handler: s.handler(),
|
||||
ReadHeaderTimeout: readHeadTimeout,
|
||||
ReadTimeout: 10 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() { errCh <- srv.Serve(ln) }()
|
||||
s.log.Info("discovery API listening", "addr", s.boundAddr)
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownGrace)
|
||||
defer cancel()
|
||||
if err := srv.Shutdown(shutdownCtx); err != nil {
|
||||
return err
|
||||
}
|
||||
<-errCh // always http.ErrServerClosed after a clean Shutdown
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// handler assembles the mux and the middleware chain, outermost first:
|
||||
// recover → request-log → body-size cap → bearer auth.
|
||||
func (s *Server) handler() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok\n"))
|
||||
})
|
||||
mux.HandleFunc("GET /v1/proxies", s.handleListProxies)
|
||||
mux.HandleFunc("POST /v1/leases", s.handleAcquireLease)
|
||||
mux.HandleFunc("DELETE /v1/leases/{id}", s.handleReleaseLease)
|
||||
mux.HandleFunc("POST /v1/leases/{id}/report", s.handleReportLease)
|
||||
|
||||
var h http.Handler = mux
|
||||
h = s.authMiddleware(h)
|
||||
h = maxBytesMiddleware(h)
|
||||
h = s.logMiddleware(h)
|
||||
h = s.recoverMiddleware(h)
|
||||
return h
|
||||
}
|
||||
|
||||
func (s *Server) recoverMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
s.log.Error(nil, "panic in discovery handler", "panic", p, "path", r.URL.Path)
|
||||
writeError(w, http.StatusInternalServerError, "internal", "internal server error")
|
||||
}
|
||||
}()
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// statusRecorder captures the response code for the request log.
|
||||
type statusRecorder struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
}
|
||||
|
||||
func (r *statusRecorder) WriteHeader(code int) {
|
||||
r.status = code
|
||||
r.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (s *Server) logMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/healthz" {
|
||||
next.ServeHTTP(w, r) // probes are noise
|
||||
return
|
||||
}
|
||||
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
|
||||
start := time.Now()
|
||||
next.ServeHTTP(rec, r)
|
||||
s.log.Info("request",
|
||||
"method", r.Method, "path", r.URL.Path,
|
||||
"status", rec.status, "duration", time.Since(start).String())
|
||||
})
|
||||
}
|
||||
|
||||
func maxBytesMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) authMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.Token == "" || r.URL.Path == "/healthz" {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
|
||||
if !ok || subtle.ConstantTimeCompare([]byte(token), []byte(s.Token)) != 1 {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "missing or invalid bearer token")
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// errorBody is the shared error shape:
|
||||
// {"error":"<machine_code>","message":"<human>"}.
|
||||
type errorBody struct {
|
||||
Error string `json:"error"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, code, message string) {
|
||||
writeJSON(w, status, errorBody{Error: code, Message: message})
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
386
internal/discovery/server_test.go
Normal file
386
internal/discovery/server_test.go
Normal file
@@ -0,0 +1,386 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
|
||||
)
|
||||
|
||||
func testProxy(name string, attrs map[string]string, healthy bool, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy {
|
||||
p := &crawlv1alpha1.Proxy{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name, Namespace: "default", UID: types.UID("uid-" + name),
|
||||
},
|
||||
Spec: crawlv1alpha1.ProxySpec{
|
||||
Mode: crawlv1alpha1.ModeExternal,
|
||||
Endpoint: &crawlv1alpha1.EndpointSpec{Host: "10.0.0.1", Port: 3128},
|
||||
Attributes: attrs,
|
||||
},
|
||||
}
|
||||
p.Status.IP = "10.0.0.1"
|
||||
p.Status.Phase = crawlv1alpha1.PhaseReady
|
||||
status := metav1.ConditionFalse
|
||||
if healthy {
|
||||
status = metav1.ConditionTrue
|
||||
}
|
||||
p.Status.Conditions = []metav1.Condition{{
|
||||
Type: crawlv1alpha1.ConditionHealthy, Status: status,
|
||||
Reason: "Probing", LastTransitionTime: metav1.Now(),
|
||||
}}
|
||||
for _, m := range mut {
|
||||
m(p)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func withMaxLeases(n int32) func(*crawlv1alpha1.Proxy) {
|
||||
return func(p *crawlv1alpha1.Proxy) { p.Spec.MaxLeases = &n }
|
||||
}
|
||||
|
||||
func withLatency(ms int64) func(*crawlv1alpha1.Proxy) {
|
||||
return func(p *crawlv1alpha1.Proxy) { p.Status.LatencyMillis = ms }
|
||||
}
|
||||
|
||||
// newTestServer wires the handler chain to a fake cache reader and a real
|
||||
// lease store, served over httptest.
|
||||
func newTestServer(t *testing.T, token string, proxies ...*crawlv1alpha1.Proxy) (*httptest.Server, *Server) {
|
||||
t.Helper()
|
||||
s := runtime.NewScheme()
|
||||
if err := crawlv1alpha1.AddToScheme(s); err != nil {
|
||||
t.Fatalf("scheme: %v", err)
|
||||
}
|
||||
builder := fake.NewClientBuilder().WithScheme(s)
|
||||
for _, p := range proxies {
|
||||
builder = builder.WithObjects(p)
|
||||
}
|
||||
srv := &Server{
|
||||
Reader: builder.Build(),
|
||||
Store: lease.NewStore(15 * time.Minute),
|
||||
Token: token,
|
||||
MaxLeaseTTL: time.Hour,
|
||||
}
|
||||
ts := httptest.NewServer(srv.handler())
|
||||
t.Cleanup(ts.Close)
|
||||
return ts, srv
|
||||
}
|
||||
|
||||
type response struct {
|
||||
status int
|
||||
body map[string]any
|
||||
}
|
||||
|
||||
func do(t *testing.T, ts *httptest.Server, method, path, token string, body any) response {
|
||||
t.Helper()
|
||||
var reader io.Reader
|
||||
if body != nil {
|
||||
if s, ok := body.(string); ok {
|
||||
reader = bytes.NewBufferString(s)
|
||||
} else {
|
||||
b, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
t.Fatalf("marshaling request body: %v", err)
|
||||
}
|
||||
reader = bytes.NewBuffer(b)
|
||||
}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(context.Background(), method, ts.URL+path, reader)
|
||||
if err != nil {
|
||||
t.Fatalf("building request: %v", err)
|
||||
}
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
resp, err := ts.Client().Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", method, path, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
out := response{status: resp.StatusCode}
|
||||
raw, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("reading response: %v", err)
|
||||
}
|
||||
if len(raw) > 0 && resp.Header.Get("Content-Type") == "application/json" {
|
||||
if err := json.Unmarshal(raw, &out.body); err != nil {
|
||||
t.Fatalf("decoding response %q: %v", raw, err)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAuth(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts, _ := newTestServer(t, "sekrit", testProxy("p1", nil, true))
|
||||
|
||||
if got := do(t, ts, http.MethodGet, "/v1/proxies", "", nil); got.status != http.StatusUnauthorized {
|
||||
t.Errorf("no token: status %d, want 401", got.status)
|
||||
}
|
||||
if got := do(t, ts, http.MethodGet, "/v1/proxies", "wrong", nil); got.status != http.StatusUnauthorized {
|
||||
t.Errorf("wrong token: status %d, want 401", got.status)
|
||||
}
|
||||
if got := do(t, ts, http.MethodGet, "/v1/proxies", "sekrit", nil); got.status != http.StatusOK {
|
||||
t.Errorf("correct token: status %d, want 200", got.status)
|
||||
}
|
||||
if got := do(t, ts, http.MethodGet, "/healthz", "", nil); got.status != http.StatusOK {
|
||||
t.Errorf("healthz without token: status %d, want 200 (always unauthenticated)", got.status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuth_disabledWithEmptyToken(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
|
||||
if got := do(t, ts, http.MethodGet, "/v1/proxies", "", nil); got.status != http.StatusOK {
|
||||
t.Errorf("status %d, want 200 with auth disabled", got.status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListProxies(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts, _ := newTestServer(t, "",
|
||||
testProxy("eu-healthy", map[string]string{"geo": "eu", "purpose": "crawl"}, true),
|
||||
testProxy("eu-sick", map[string]string{"geo": "eu"}, false),
|
||||
testProxy("us-healthy", map[string]string{"geo": "us"}, true),
|
||||
)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
wantCount int
|
||||
wantFirst string
|
||||
}{
|
||||
{name: "no filter returns everything", query: "", wantCount: 3, wantFirst: "default/eu-healthy"},
|
||||
{name: "healthy filter", query: "?healthy=true", wantCount: 2},
|
||||
{name: "unhealthy filter", query: "?healthy=false", wantCount: 1, wantFirst: "default/eu-sick"},
|
||||
{name: "attribute filter", query: "?attr.geo=eu", wantCount: 2},
|
||||
{name: "attribute and health combined", query: "?attr.geo=eu&healthy=true", wantCount: 1, wantFirst: "default/eu-healthy"},
|
||||
{name: "two attributes must both match", query: "?attr.geo=eu&attr.purpose=crawl", wantCount: 1, wantFirst: "default/eu-healthy"},
|
||||
{name: "no matches is 200 with count 0", query: "?attr.geo=mars", wantCount: 0},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := do(t, ts, http.MethodGet, "/v1/proxies"+tc.query, "", nil)
|
||||
if got.status != http.StatusOK {
|
||||
t.Fatalf("status %d, want 200", got.status)
|
||||
}
|
||||
count := int(got.body["count"].(float64))
|
||||
proxies := got.body["proxies"].([]any)
|
||||
if count != tc.wantCount || len(proxies) != tc.wantCount {
|
||||
t.Fatalf("count = %d (len %d), want %d", count, len(proxies), tc.wantCount)
|
||||
}
|
||||
if tc.wantFirst != "" {
|
||||
first := proxies[0].(map[string]any)
|
||||
if first["id"] != tc.wantFirst {
|
||||
t.Errorf("first id = %v, want %s", first["id"], tc.wantFirst)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("invalid healthy value is 400", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := do(t, ts, http.MethodGet, "/v1/proxies?healthy=maybe", "", nil); got.status != http.StatusBadRequest {
|
||||
t.Errorf("status %d, want 400", got.status)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAcquireLease_grantShape(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts, _ := newTestServer(t, "",
|
||||
testProxy("eu1", map[string]string{"geo": "eu"}, true, withLatency(30)),
|
||||
testProxy("eu2", map[string]string{"geo": "eu"}, true, withLatency(10)),
|
||||
)
|
||||
|
||||
got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
|
||||
"selector": map[string]string{"geo": "eu"},
|
||||
})
|
||||
if got.status != http.StatusCreated {
|
||||
t.Fatalf("status %d (%v), want 201", got.status, got.body)
|
||||
}
|
||||
if got.body["leaseID"] == "" || got.body["leaseID"] == nil {
|
||||
t.Error("empty leaseID")
|
||||
}
|
||||
if got.body["ttlSeconds"].(float64) != 300 {
|
||||
t.Errorf("ttlSeconds = %v, want the 300 default", got.body["ttlSeconds"])
|
||||
}
|
||||
proxy := got.body["proxy"].(map[string]any)
|
||||
if proxy["id"] != "default/eu2" {
|
||||
t.Errorf("granted %v, want default/eu2 (lower latency at equal load)", proxy["id"])
|
||||
}
|
||||
if proxy["activeLeases"].(float64) != 1 {
|
||||
t.Errorf("activeLeases = %v, want 1 (this grant included)", proxy["activeLeases"])
|
||||
}
|
||||
if _, err := time.Parse(time.RFC3339, got.body["expiresAt"].(string)); err != nil {
|
||||
t.Errorf("expiresAt %v is not RFC3339: %v", got.body["expiresAt"], err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireLease_noMatchBody(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts, _ := newTestServer(t, "",
|
||||
testProxy("eu-tiny", map[string]string{"geo": "eu"}, true, withMaxLeases(1)),
|
||||
testProxy("eu-sick", map[string]string{"geo": "eu"}, false),
|
||||
)
|
||||
|
||||
body := map[string]any{"selector": map[string]string{"geo": "eu"}}
|
||||
if got := do(t, ts, http.MethodPost, "/v1/leases", "", body); got.status != http.StatusCreated {
|
||||
t.Fatalf("first acquire: status %d, want 201", got.status)
|
||||
}
|
||||
|
||||
got := do(t, ts, http.MethodPost, "/v1/leases", "", body)
|
||||
if got.status != http.StatusConflict {
|
||||
t.Fatalf("second acquire: status %d, want 409", got.status)
|
||||
}
|
||||
want := map[string]float64{"considered": 2, "atCapacity": 1, "inCooldown": 0, "unhealthy": 1}
|
||||
for k, v := range want {
|
||||
if got.body[k].(float64) != v {
|
||||
t.Errorf("%s = %v, want %v (body %v)", k, got.body[k], v, got.body)
|
||||
}
|
||||
}
|
||||
if got.body["error"] != "no_match" {
|
||||
t.Errorf("error = %v, want no_match", got.body["error"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireLease_badRequests(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
body any
|
||||
wantCode string
|
||||
}{
|
||||
{name: "ttl above the cap", body: map[string]any{"ttlSeconds": 999999}, wantCode: "invalid_ttl"},
|
||||
{name: "negative ttl", body: map[string]any{"ttlSeconds": -5}, wantCode: "invalid_ttl"},
|
||||
{name: "malformed json", body: "{not json", wantCode: "invalid_body"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := do(t, ts, http.MethodPost, "/v1/leases", "", tc.body)
|
||||
if got.status != http.StatusBadRequest || got.body["error"] != tc.wantCode {
|
||||
t.Errorf("= %d/%v, want 400/%s", got.status, got.body["error"], tc.wantCode)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseLease_alwaysNoContent(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
|
||||
|
||||
got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{})
|
||||
if got.status != http.StatusCreated {
|
||||
t.Fatalf("acquire: status %d, want 201", got.status)
|
||||
}
|
||||
id := got.body["leaseID"].(string)
|
||||
|
||||
for _, path := range []string{"/v1/leases/" + id, "/v1/leases/" + id, "/v1/leases/never-existed"} {
|
||||
if got := do(t, ts, http.MethodDelete, path, "", nil); got.status != http.StatusNoContent {
|
||||
t.Errorf("DELETE %s: status %d, want 204", path, got.status)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportLease(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts, _ := newTestServer(t, "", testProxy("p1", map[string]string{"geo": "eu"}, true))
|
||||
|
||||
got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
|
||||
"selector": map[string]string{"geo": "eu"}, "target": "example.com",
|
||||
})
|
||||
if got.status != http.StatusCreated {
|
||||
t.Fatalf("acquire: status %d, want 201", got.status)
|
||||
}
|
||||
id := got.body["leaseID"].(string)
|
||||
reportPath := fmt.Sprintf("/v1/leases/%s/report", id)
|
||||
|
||||
if got := do(t, ts, http.MethodPost, reportPath, "", map[string]any{"result": "rate_limited", "target": "example.com"}); got.status != http.StatusNoContent {
|
||||
t.Fatalf("report: status %d, want 204", got.status)
|
||||
}
|
||||
|
||||
// The cooldown from the report now blocks same-target acquisition.
|
||||
got = do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
|
||||
"selector": map[string]string{"geo": "eu"}, "target": "example.com",
|
||||
})
|
||||
if got.status != http.StatusConflict || got.body["inCooldown"].(float64) != 1 {
|
||||
t.Errorf("post-report acquire = %d/%v, want 409 with inCooldown 1", got.status, got.body)
|
||||
}
|
||||
|
||||
t.Run("invalid result value", func(t *testing.T) {
|
||||
got := do(t, ts, http.MethodPost, reportPath, "", map[string]any{"result": "throttled"})
|
||||
if got.status != http.StatusBadRequest || got.body["error"] != "invalid_result" {
|
||||
t.Errorf("= %d/%v, want 400/invalid_result", got.status, got.body["error"])
|
||||
}
|
||||
})
|
||||
t.Run("unknown lease", func(t *testing.T) {
|
||||
got := do(t, ts, http.MethodPost, "/v1/leases/never-existed/report", "", map[string]any{"result": "ok"})
|
||||
if got.status != http.StatusNotFound || got.body["error"] != "unknown_lease" {
|
||||
t.Errorf("= %d/%v, want 404/unknown_lease", got.status, got.body["error"])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestStart_servesAndShutsDown(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := runtime.NewScheme()
|
||||
if err := crawlv1alpha1.AddToScheme(s); err != nil {
|
||||
t.Fatalf("scheme: %v", err)
|
||||
}
|
||||
srv := &Server{
|
||||
Reader: fake.NewClientBuilder().WithScheme(s).Build(),
|
||||
Store: lease.NewStore(time.Minute),
|
||||
Addr: "127.0.0.1:0",
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- srv.Start(ctx) }()
|
||||
|
||||
var addr string
|
||||
deadline := time.After(5 * time.Second)
|
||||
for addr == "" {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("server never bound")
|
||||
case <-time.After(5 * time.Millisecond):
|
||||
addr = srv.BoundAddr()
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := http.Get("http://" + addr + "/healthz")
|
||||
if err != nil {
|
||||
t.Fatalf("healthz: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("healthz status %d, want 200", resp.StatusCode)
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Errorf("Start returned %v, want nil after graceful shutdown", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Start did not stop on cancel")
|
||||
}
|
||||
}
|
||||
125
internal/gc/gc.go
Normal file
125
internal/gc/gc.go
Normal file
@@ -0,0 +1,125 @@
|
||||
// Package gc implements orphan garbage collection: a periodic sweep that
|
||||
// deletes provider instances tagged by this operator whose owning Proxy CR
|
||||
// no longer exists — the safety net for crashes between a provider Create
|
||||
// and the status write that records it.
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// Sweeper is the manager Runnable running the sweep loop.
|
||||
type Sweeper struct {
|
||||
// Reader lists Proxies from the manager's cache to establish the live
|
||||
// UID set.
|
||||
Reader client.Reader
|
||||
// Providers are the configured backends; each is swept independently.
|
||||
Providers map[string]provider.Provider
|
||||
|
||||
// Interval between sweeps (default 10m). The first sweep runs one full
|
||||
// interval after start, not immediately — right after startup the
|
||||
// cache is coldest and an in-flight create is most likely.
|
||||
Interval time.Duration
|
||||
// MinAge exempts young instances (default 10m): an instance mid-create
|
||||
// may not have its status write landed yet; deleting it would race the
|
||||
// reconciler.
|
||||
MinAge time.Duration
|
||||
|
||||
// NamespaceRestricted must be set when the manager cache is limited to
|
||||
// one namespace. Then the live-UID set is incomplete, and a sweep
|
||||
// would delete VMs owned by Proxies the cache cannot see — so Start
|
||||
// refuses unless AllowNamespaced (--gc-allow-namespaced) is explicit.
|
||||
NamespaceRestricted bool
|
||||
AllowNamespaced bool
|
||||
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// NeedLeaderElection is true: the sweep is destructive and must have a
|
||||
// single writer.
|
||||
func (s *Sweeper) NeedLeaderElection() bool { return true }
|
||||
|
||||
// Start runs the sweep loop until ctx ends.
|
||||
func (s *Sweeper) Start(ctx context.Context) error {
|
||||
if s.NamespaceRestricted && !s.AllowNamespaced {
|
||||
return errors.New(
|
||||
"orphan GC refuses to run against a namespace-restricted cache: proxies outside the namespace " +
|
||||
"would count as orphans and their instances would be deleted; pass --gc-allow-namespaced to override")
|
||||
}
|
||||
if s.Interval == 0 {
|
||||
s.Interval = 10 * time.Minute
|
||||
}
|
||||
if s.MinAge == 0 {
|
||||
s.MinAge = 10 * time.Minute
|
||||
}
|
||||
if s.now == nil {
|
||||
s.now = time.Now
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(s.Interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
s.sweep(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sweep deletes tagged instances whose UID matches no existing Proxy CR.
|
||||
// A CR with a deletionTimestamp still counts as live: its finalizer owns
|
||||
// that deletion, and GC racing it would double-delete. A UID becomes
|
||||
// orphan-eligible only once the object is fully gone.
|
||||
func (s *Sweeper) sweep(ctx context.Context) {
|
||||
log := logf.FromContext(ctx).WithName("orphan-gc")
|
||||
|
||||
var list crawlv1alpha1.ProxyList
|
||||
if err := s.Reader.List(ctx, &list); err != nil {
|
||||
// Without the live set nothing can be proven orphaned; skip the
|
||||
// whole sweep rather than guess.
|
||||
log.Error(err, "listing proxies; skipping this sweep")
|
||||
return
|
||||
}
|
||||
live := make(map[string]bool, len(list.Items))
|
||||
for i := range list.Items {
|
||||
live[string(list.Items[i].UID)] = true
|
||||
}
|
||||
|
||||
for name, prov := range s.Providers {
|
||||
instances, err := prov.ListByTag(ctx)
|
||||
if err != nil {
|
||||
// One broken provider must not abort the sweep for the rest.
|
||||
log.Error(err, "listing instances; skipping this provider", "provider", name)
|
||||
continue
|
||||
}
|
||||
for _, inst := range instances {
|
||||
switch {
|
||||
case inst.UID == "":
|
||||
// Managed label without a UID label shouldn't exist for
|
||||
// anything this operator created; without ownership proof,
|
||||
// never delete.
|
||||
continue
|
||||
case live[inst.UID]:
|
||||
continue
|
||||
case s.now().Sub(inst.CreatedAt) < s.MinAge:
|
||||
continue
|
||||
}
|
||||
log.Info("WARNING: deleting orphaned instance",
|
||||
"provider", name, "providerID", inst.ID, "uid", inst.UID)
|
||||
if err := prov.Delete(ctx, inst.ID); err != nil {
|
||||
log.Error(err, "deleting orphaned instance",
|
||||
"provider", name, "providerID", inst.ID, "uid", inst.UID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
214
internal/gc/gc_test.go
Normal file
214
internal/gc/gc_test.go
Normal file
@@ -0,0 +1,214 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// listProvider serves a canned instance list and records deletions.
|
||||
type listProvider struct {
|
||||
mu sync.Mutex
|
||||
instances []provider.Instance
|
||||
listErr error
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (l *listProvider) Create(context.Context, provider.CreateRequest) (string, error) {
|
||||
return "", errors.New("not used")
|
||||
}
|
||||
|
||||
func (l *listProvider) Get(context.Context, string) (*provider.Instance, error) {
|
||||
return nil, provider.ErrNotFound
|
||||
}
|
||||
|
||||
func (l *listProvider) Delete(_ context.Context, id string) error {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.deleted = append(l.deleted, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *listProvider) ListByTag(context.Context) ([]provider.Instance, error) {
|
||||
return l.instances, l.listErr
|
||||
}
|
||||
|
||||
func (l *listProvider) deletedIDs() []string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return append([]string(nil), l.deleted...)
|
||||
}
|
||||
|
||||
func proxyWithUID(name, uid string, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy {
|
||||
p := &crawlv1alpha1.Proxy{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", UID: types.UID(uid)},
|
||||
Spec: crawlv1alpha1.ProxySpec{
|
||||
Mode: crawlv1alpha1.ModeManaged,
|
||||
Provider: "stub",
|
||||
},
|
||||
}
|
||||
for _, m := range mut {
|
||||
m(p)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func newReader(t *testing.T, objs ...client.Object) client.Reader {
|
||||
t.Helper()
|
||||
s := runtime.NewScheme()
|
||||
if err := crawlv1alpha1.AddToScheme(s); err != nil {
|
||||
t.Fatalf("scheme: %v", err)
|
||||
}
|
||||
return fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build()
|
||||
}
|
||||
|
||||
func oldInstance(id, uid string) provider.Instance {
|
||||
return provider.Instance{ID: id, UID: uid, State: provider.StateRunning,
|
||||
CreatedAt: time.Now().Add(-time.Hour)}
|
||||
}
|
||||
|
||||
func newSweeper(reader client.Reader, providers map[string]provider.Provider) *Sweeper {
|
||||
return &Sweeper{
|
||||
Reader: reader,
|
||||
Providers: providers,
|
||||
Interval: 10 * time.Minute,
|
||||
MinAge: 10 * time.Minute,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweep_deletesOnlyTrueOrphans(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
deletingCR := proxyWithUID("deleting", "uid-deleting", func(p *crawlv1alpha1.Proxy) {
|
||||
now := metav1.Now()
|
||||
p.DeletionTimestamp = &now
|
||||
p.Finalizers = []string{crawlv1alpha1.FinalizerName}
|
||||
})
|
||||
prov := &listProvider{instances: []provider.Instance{
|
||||
oldInstance("inst-live", "uid-live"),
|
||||
oldInstance("inst-orphan", "uid-orphan"),
|
||||
oldInstance("inst-deleting", "uid-deleting"),
|
||||
{ID: "inst-young", UID: "uid-young-orphan", State: provider.StateRunning,
|
||||
CreatedAt: time.Now().Add(-time.Minute)},
|
||||
oldInstance("inst-unlabelled", ""),
|
||||
}}
|
||||
s := newSweeper(
|
||||
newReader(t, proxyWithUID("live", "uid-live"), deletingCR),
|
||||
map[string]provider.Provider{"stub": prov},
|
||||
)
|
||||
|
||||
s.sweep(context.Background())
|
||||
|
||||
got := prov.deletedIDs()
|
||||
if len(got) != 1 || got[0] != "inst-orphan" {
|
||||
t.Errorf("deleted %v, want exactly [inst-orphan]:\n"+
|
||||
"live CR's instance must stay; a deleting CR still owns its instance (finalizer, not GC);\n"+
|
||||
"young instances may be mid-create; unlabelled instances have no ownership proof", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweep_providerErrorDoesNotAbortOthers(t *testing.T) {
|
||||
t.Parallel()
|
||||
broken := &listProvider{listErr: errors.New("cloud is down")}
|
||||
working := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
|
||||
s := newSweeper(newReader(t), map[string]provider.Provider{
|
||||
"broken": broken,
|
||||
"working": working,
|
||||
})
|
||||
|
||||
s.sweep(context.Background())
|
||||
|
||||
if got := working.deletedIDs(); len(got) != 1 {
|
||||
t.Errorf("working provider deletions = %v, want the orphan despite the broken provider", got)
|
||||
}
|
||||
}
|
||||
|
||||
// errReader fails every List: without the live set nothing can be proven
|
||||
// orphaned, so the sweep must delete nothing.
|
||||
type errReader struct{ client.Reader }
|
||||
|
||||
func (errReader) List(context.Context, client.ObjectList, ...client.ListOption) error {
|
||||
return errors.New("cache broken")
|
||||
}
|
||||
|
||||
func TestSweep_listFailureSkipsSweep(t *testing.T) {
|
||||
t.Parallel()
|
||||
prov := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
|
||||
s := newSweeper(errReader{}, map[string]provider.Provider{"stub": prov})
|
||||
|
||||
s.sweep(context.Background())
|
||||
|
||||
if got := prov.deletedIDs(); len(got) != 0 {
|
||||
t.Errorf("deleted %v with an unreadable live set, want nothing", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStart_namespaceGuard(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newSweeper(newReader(t), nil)
|
||||
s.NamespaceRestricted = true
|
||||
err := s.Start(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "--gc-allow-namespaced") {
|
||||
t.Errorf("Start with restricted cache = %v, want refusal naming the override flag", err)
|
||||
}
|
||||
|
||||
s2 := newSweeper(newReader(t), nil)
|
||||
s2.NamespaceRestricted = true
|
||||
s2.AllowNamespaced = true
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s2.Start(ctx) }()
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Errorf("Start with override = %v, want it to run until cancel", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Start did not stop on cancel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStart_sweepsOnIntervalAndStops(t *testing.T) {
|
||||
t.Parallel()
|
||||
prov := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
|
||||
s := newSweeper(newReader(t), map[string]provider.Provider{"stub": prov})
|
||||
s.Interval = 5 * time.Millisecond
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s.Start(ctx) }()
|
||||
|
||||
deadline := time.After(5 * time.Second)
|
||||
for len(prov.deletedIDs()) == 0 {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("no sweep ran")
|
||||
case <-time.After(2 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Errorf("Start = %v, want nil", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Start did not stop on cancel")
|
||||
}
|
||||
}
|
||||
@@ -62,6 +62,14 @@ type probeJob struct {
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// ProbeMetrics receives every probe result and the retirement of a
|
||||
// proxy's series. Implemented by internal/metrics; defined here so this
|
||||
// package carries no metrics dependency.
|
||||
type ProbeMetrics interface {
|
||||
ObserveProbe(proxy string, latency time.Duration, success bool)
|
||||
ForgetProxy(proxy string)
|
||||
}
|
||||
|
||||
// Engine runs the probe scheduler and worker pool as a manager Runnable. It
|
||||
// never writes Proxy status itself — keeping the reconciler the single
|
||||
// status writer — and instead emits a GenericEvent per status-affecting
|
||||
@@ -87,6 +95,10 @@ type Engine struct {
|
||||
// ProbeTLSConfig overrides TLS verification for https probe URLs; nil
|
||||
// means system roots. Needed for private CAs (and tests).
|
||||
ProbeTLSConfig *tls.Config
|
||||
// Metrics, when non-nil, is fed on every probe — the status writes are
|
||||
// transition-only by design, so metrics are where high-frequency
|
||||
// signal (true probe recency, every latency sample) lives.
|
||||
Metrics ProbeMetrics
|
||||
|
||||
probeFn func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult
|
||||
|
||||
@@ -220,6 +232,11 @@ func (e *Engine) tick(ctx context.Context, now time.Time, jobs chan<- probeJob)
|
||||
for key := range e.states {
|
||||
if _, ok := probeable[key]; !ok {
|
||||
delete(e.states, key)
|
||||
if e.Metrics != nil {
|
||||
// Retire the per-proxy series with the state, or series
|
||||
// for deleted proxies leak forever.
|
||||
e.Metrics.ForgetProxy(key.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -252,6 +269,10 @@ func newState(p *crawlv1alpha1.Proxy, now time.Time, interval time.Duration) *st
|
||||
// threshold-crossing flip, or a material latency change (beyond
|
||||
// max(LatencyFloor, 50% of reported) and rate-limited by MinReportInterval).
|
||||
func (e *Engine) record(job probeJob, res probeResult, now time.Time) {
|
||||
if e.Metrics != nil {
|
||||
e.Metrics.ObserveProbe(job.key.String(), res.latency, res.ok)
|
||||
}
|
||||
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
|
||||
|
||||
319
internal/lease/store.go
Normal file
319
internal/lease/store.go
Normal file
@@ -0,0 +1,319 @@
|
||||
// Package lease implements the in-memory lease store behind the discovery
|
||||
// API: TTL-based proxy assignment with server-side usage tracking and
|
||||
// per-(proxy, target) cooldowns. Accepted prototype limitation, documented
|
||||
// in the README: state is per-process, so an operator restart drops all
|
||||
// leases and cooldowns — clients must tolerate a lease vanishing (their
|
||||
// requests still work; they just re-lease).
|
||||
package lease
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Result is a client's report of how a leased proxy behaved against a
|
||||
// target. ResultRateLimited and ResultBanned record a cooldown; ResultOK is
|
||||
// an acknowledgement and records nothing.
|
||||
type Result string
|
||||
|
||||
const (
|
||||
ResultOK Result = "ok"
|
||||
ResultRateLimited Result = "rate_limited"
|
||||
ResultBanned Result = "banned"
|
||||
)
|
||||
|
||||
// ParseResult maps a wire value to a Result; ok is false for anything
|
||||
// unknown, which the API layer turns into a 400.
|
||||
func ParseResult(s string) (Result, bool) {
|
||||
switch r := Result(s); r {
|
||||
case ResultOK, ResultRateLimited, ResultBanned:
|
||||
return r, true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrNoMatch means no candidate could take a lease; AcquireStats says
|
||||
// why, and the API layer turns both into the 409 body.
|
||||
ErrNoMatch = errors.New("lease: no candidate available")
|
||||
// ErrUnknownLease means the lease ID does not resolve (404). Reports on
|
||||
// recently expired leases do NOT hit this — see the retention note on
|
||||
// Store.
|
||||
ErrUnknownLease = errors.New("lease: unknown lease id")
|
||||
)
|
||||
|
||||
// Candidate is one leasable proxy as seen by the caller at selection time.
|
||||
// The store itself knows nothing about Proxy objects — the discovery layer
|
||||
// filters for health/attributes and passes what selection needs.
|
||||
type Candidate struct {
|
||||
// Proxy is the opaque proxy key ("namespace/name").
|
||||
Proxy string
|
||||
// MaxLeases caps concurrent leases; 0 means unleasable.
|
||||
MaxLeases int32
|
||||
// Latency is the proxy's last reported latency, used as the tie-break.
|
||||
Latency time.Duration
|
||||
}
|
||||
|
||||
// Lease is a granted assignment. Values returned by the store are copies;
|
||||
// mutating them does not affect the store.
|
||||
type Lease struct {
|
||||
ID string
|
||||
Proxy string
|
||||
Target string
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// AcquireRequest carries the candidate set and lease parameters. Acquire
|
||||
// deliberately takes the whole candidate set, not a pre-chosen proxy:
|
||||
// selection and insertion must happen under one lock, or two concurrent
|
||||
// requests both see "3 of 5 used" and overcommit.
|
||||
type AcquireRequest struct {
|
||||
Candidates []Candidate
|
||||
// Target scopes the cooldown check; empty means the global pool.
|
||||
Target string
|
||||
TTL time.Duration
|
||||
}
|
||||
|
||||
// AcquireStats explains an ErrNoMatch (and is returned on success too):
|
||||
// every candidate is either leased, at capacity, or in cooldown.
|
||||
type AcquireStats struct {
|
||||
Considered int
|
||||
AtCapacity int
|
||||
InCooldown int
|
||||
}
|
||||
|
||||
type cooldownKey struct{ proxy, target string }
|
||||
|
||||
// Store is the in-memory lease store. One mutex guards everything: at tens
|
||||
// of proxies and human-rate QPS, sharding would be premature complexity.
|
||||
//
|
||||
// Retention: an expired lease is kept for CooldownWindow past its TTL so a
|
||||
// Report arriving just after expiry still resolves — which matters most
|
||||
// exactly when a proxy is being rate-limited. Acquire and the counts ignore
|
||||
// retained leases; only the sweep finally drops them.
|
||||
type Store struct {
|
||||
// CooldownWindow is how long a reported proxy/target pair is excluded
|
||||
// from selection (default 15m; --lease-cooldown in Step 10).
|
||||
CooldownWindow time.Duration
|
||||
// SweepInterval is how often the expiry sweep runs (default 30s).
|
||||
SweepInterval time.Duration
|
||||
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
byID map[string]*Lease
|
||||
byProxy map[string]map[string]*Lease
|
||||
cooldowns map[cooldownKey]time.Time
|
||||
}
|
||||
|
||||
// NewStore returns a ready Store. A non-positive cooldownWindow selects the
|
||||
// 15-minute default.
|
||||
func NewStore(cooldownWindow time.Duration) *Store {
|
||||
if cooldownWindow <= 0 {
|
||||
cooldownWindow = 15 * time.Minute
|
||||
}
|
||||
return &Store{
|
||||
CooldownWindow: cooldownWindow,
|
||||
SweepInterval: 30 * time.Second,
|
||||
now: time.Now,
|
||||
byID: map[string]*Lease{},
|
||||
byProxy: map[string]map[string]*Lease{},
|
||||
cooldowns: map[cooldownKey]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
// Acquire selects the least-loaded eligible candidate (ties: lowest
|
||||
// latency, then name, so selection is deterministic and testable) and
|
||||
// grants a lease on it.
|
||||
func (s *Store) Acquire(_ context.Context, req AcquireRequest) (*Lease, AcquireStats, error) {
|
||||
stats := AcquireStats{Considered: len(req.Candidates)}
|
||||
if req.TTL <= 0 {
|
||||
return nil, stats, fmt.Errorf("lease: non-positive TTL %v", req.TTL)
|
||||
}
|
||||
now := s.now()
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
type eligible struct {
|
||||
cand Candidate
|
||||
active int
|
||||
}
|
||||
var elig []eligible
|
||||
for _, c := range req.Candidates {
|
||||
if s.inCooldownLocked(c.Proxy, req.Target, now) {
|
||||
stats.InCooldown++
|
||||
continue
|
||||
}
|
||||
active := s.activeCountLocked(c.Proxy, now)
|
||||
if int32(active) >= c.MaxLeases {
|
||||
stats.AtCapacity++
|
||||
continue
|
||||
}
|
||||
elig = append(elig, eligible{cand: c, active: active})
|
||||
}
|
||||
if len(elig) == 0 {
|
||||
return nil, stats, ErrNoMatch
|
||||
}
|
||||
|
||||
slices.SortFunc(elig, func(a, b eligible) int {
|
||||
if c := cmp.Compare(a.active, b.active); c != 0 {
|
||||
return c
|
||||
}
|
||||
if c := cmp.Compare(a.cand.Latency, b.cand.Latency); c != 0 {
|
||||
return c
|
||||
}
|
||||
return strings.Compare(a.cand.Proxy, b.cand.Proxy)
|
||||
})
|
||||
|
||||
l := &Lease{
|
||||
ID: rand.Text(),
|
||||
Proxy: elig[0].cand.Proxy,
|
||||
Target: req.Target,
|
||||
ExpiresAt: now.Add(req.TTL),
|
||||
}
|
||||
s.byID[l.ID] = l
|
||||
if s.byProxy[l.Proxy] == nil {
|
||||
s.byProxy[l.Proxy] = map[string]*Lease{}
|
||||
}
|
||||
s.byProxy[l.Proxy][l.ID] = l
|
||||
|
||||
granted := *l
|
||||
return &granted, stats, nil
|
||||
}
|
||||
|
||||
// Release drops a lease early. Idempotent: releasing an unknown or already
|
||||
// expired lease is a no-op, so the API's DELETE can always answer 204.
|
||||
func (s *Store) Release(_ context.Context, id string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.dropLocked(id)
|
||||
}
|
||||
|
||||
// Report records the outcome of using a lease. Rate-limited and banned
|
||||
// results put the (proxy, target) pair in cooldown — target taken from the
|
||||
// report, falling back to the lease's own target, falling back to the
|
||||
// global pool. Reports on recently expired leases still resolve (see the
|
||||
// retention note on Store).
|
||||
func (s *Store) Report(_ context.Context, id string, result Result, target string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
l, ok := s.byID[id]
|
||||
if !ok {
|
||||
return ErrUnknownLease
|
||||
}
|
||||
if result == ResultOK {
|
||||
return nil
|
||||
}
|
||||
if target == "" {
|
||||
target = l.Target
|
||||
}
|
||||
s.cooldowns[cooldownKey{proxy: l.Proxy, target: target}] = s.now().Add(s.CooldownWindow)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ActiveCount returns the number of unexpired leases held on one proxy.
|
||||
func (s *Store) ActiveCount(proxy string) int {
|
||||
now := s.now()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.activeCountLocked(proxy, now)
|
||||
}
|
||||
|
||||
// Counts returns the active-lease count per proxy, for the discovery list
|
||||
// endpoint and the metrics collector. Proxies with no active leases are
|
||||
// absent from the map.
|
||||
func (s *Store) Counts() map[string]int {
|
||||
now := s.now()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
counts := make(map[string]int, len(s.byProxy))
|
||||
for proxy := range s.byProxy {
|
||||
if n := s.activeCountLocked(proxy, now); n > 0 {
|
||||
counts[proxy] = n
|
||||
}
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
// Start runs the expiry sweep until ctx ends; it satisfies
|
||||
// manager.Runnable so cmd/main.go can mgr.Add the store directly.
|
||||
func (s *Store) Start(ctx context.Context) error {
|
||||
ticker := time.NewTicker(s.SweepInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
s.sweep(s.now())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NeedLeaderElection is false: lease state is per-process and the discovery
|
||||
// API serves wherever this process runs, so the sweep must run there too.
|
||||
func (s *Store) NeedLeaderElection() bool { return false }
|
||||
|
||||
// sweep drops leases past their retention window and elapsed cooldowns.
|
||||
// Correctness never depends on sweep timing — every read path checks
|
||||
// expiry against the clock — so this is purely garbage collection.
|
||||
func (s *Store) sweep(now time.Time) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for id, l := range s.byID {
|
||||
if now.After(l.ExpiresAt.Add(s.CooldownWindow)) {
|
||||
s.dropLocked(id)
|
||||
}
|
||||
}
|
||||
for k, until := range s.cooldowns {
|
||||
if now.After(until) {
|
||||
delete(s.cooldowns, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) dropLocked(id string) {
|
||||
l, ok := s.byID[id]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
delete(s.byID, id)
|
||||
delete(s.byProxy[l.Proxy], id)
|
||||
if len(s.byProxy[l.Proxy]) == 0 {
|
||||
delete(s.byProxy, l.Proxy)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) activeCountLocked(proxy string, now time.Time) int {
|
||||
n := 0
|
||||
for _, l := range s.byProxy[proxy] {
|
||||
if now.Before(l.ExpiresAt) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// inCooldownLocked: the global cooldown (empty target) always applies; a
|
||||
// target-scoped cooldown additionally applies to acquisitions for that
|
||||
// target. An acquisition without a target sees only the global pool — a
|
||||
// proxy rate-limited by one site is still fine for everyone else.
|
||||
func (s *Store) inCooldownLocked(proxy, target string, now time.Time) bool {
|
||||
if until, ok := s.cooldowns[cooldownKey{proxy: proxy}]; ok && now.Before(until) {
|
||||
return true
|
||||
}
|
||||
if target == "" {
|
||||
return false
|
||||
}
|
||||
until, ok := s.cooldowns[cooldownKey{proxy: proxy, target: target}]
|
||||
return ok && now.Before(until)
|
||||
}
|
||||
365
internal/lease/store_test.go
Normal file
365
internal/lease/store_test.go
Normal file
@@ -0,0 +1,365 @@
|
||||
package lease
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeClock is an injectable, manually advanced clock.
|
||||
type fakeClock struct {
|
||||
mu sync.Mutex
|
||||
cur time.Time
|
||||
}
|
||||
|
||||
func newFakeClock() *fakeClock {
|
||||
return &fakeClock{cur: time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC)}
|
||||
}
|
||||
|
||||
func (c *fakeClock) Now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.cur
|
||||
}
|
||||
|
||||
func (c *fakeClock) Advance(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.cur = c.cur.Add(d)
|
||||
}
|
||||
|
||||
func newTestStore() (*Store, *fakeClock) {
|
||||
s := NewStore(15 * time.Minute)
|
||||
clock := newFakeClock()
|
||||
s.now = clock.Now
|
||||
return s, clock
|
||||
}
|
||||
|
||||
func candidate(proxy string, maxLeases int32, latency time.Duration) Candidate {
|
||||
return Candidate{Proxy: proxy, MaxLeases: maxLeases, Latency: latency}
|
||||
}
|
||||
|
||||
func mustAcquire(t *testing.T, s *Store, req AcquireRequest) *Lease {
|
||||
t.Helper()
|
||||
l, _, err := s.Acquire(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Acquire: %v", err)
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func TestAcquire_capacity(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, _ := newTestStore()
|
||||
req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 2, 0)}, TTL: time.Minute}
|
||||
|
||||
l1 := mustAcquire(t, s, req)
|
||||
l2 := mustAcquire(t, s, req)
|
||||
if l1.ID == l2.ID {
|
||||
t.Fatal("two leases share an ID")
|
||||
}
|
||||
if got := s.ActiveCount("ns/p1"); got != 2 {
|
||||
t.Fatalf("ActiveCount = %d, want 2", got)
|
||||
}
|
||||
|
||||
_, stats, err := s.Acquire(context.Background(), req)
|
||||
if !errors.Is(err, ErrNoMatch) {
|
||||
t.Fatalf("third acquire error = %v, want ErrNoMatch", err)
|
||||
}
|
||||
want := AcquireStats{Considered: 1, AtCapacity: 1}
|
||||
if stats != want {
|
||||
t.Errorf("stats = %+v, want %+v", stats, want)
|
||||
}
|
||||
|
||||
// Early release frees the slot again.
|
||||
s.Release(context.Background(), l1.ID)
|
||||
mustAcquire(t, s, req)
|
||||
}
|
||||
|
||||
func TestAcquire_maxLeasesZeroIsUnleasable(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, _ := newTestStore()
|
||||
_, stats, err := s.Acquire(context.Background(), AcquireRequest{
|
||||
Candidates: []Candidate{candidate("ns/p1", 0, 0)},
|
||||
TTL: time.Minute,
|
||||
})
|
||||
if !errors.Is(err, ErrNoMatch) {
|
||||
t.Fatalf("err = %v, want ErrNoMatch", err)
|
||||
}
|
||||
if stats.AtCapacity != 1 {
|
||||
t.Errorf("stats = %+v, want the unleasable proxy counted AtCapacity", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquire_selectionOrder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("least loaded wins", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, _ := newTestStore()
|
||||
mustAcquire(t, s, AcquireRequest{
|
||||
Candidates: []Candidate{candidate("ns/a", 5, 10*time.Millisecond)}, TTL: time.Minute,
|
||||
})
|
||||
l := mustAcquire(t, s, AcquireRequest{
|
||||
Candidates: []Candidate{
|
||||
candidate("ns/a", 5, 10*time.Millisecond), // 1 active, lower latency
|
||||
candidate("ns/b", 5, 90*time.Millisecond), // 0 active
|
||||
},
|
||||
TTL: time.Minute,
|
||||
})
|
||||
if l.Proxy != "ns/b" {
|
||||
t.Errorf("chose %s, want the least-loaded ns/b", l.Proxy)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("latency breaks the load tie", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, _ := newTestStore()
|
||||
l := mustAcquire(t, s, AcquireRequest{
|
||||
Candidates: []Candidate{
|
||||
candidate("ns/a", 5, 90*time.Millisecond),
|
||||
candidate("ns/b", 5, 10*time.Millisecond),
|
||||
},
|
||||
TTL: time.Minute,
|
||||
})
|
||||
if l.Proxy != "ns/b" {
|
||||
t.Errorf("chose %s, want the lower-latency ns/b", l.Proxy)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("name breaks a full tie deterministically", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, _ := newTestStore()
|
||||
l := mustAcquire(t, s, AcquireRequest{
|
||||
Candidates: []Candidate{
|
||||
candidate("ns/b", 5, 10*time.Millisecond),
|
||||
candidate("ns/a", 5, 10*time.Millisecond),
|
||||
},
|
||||
TTL: time.Minute,
|
||||
})
|
||||
if l.Proxy != "ns/a" {
|
||||
t.Errorf("chose %s, want ns/a (lexicographic tie-break)", l.Proxy)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAcquire_cooldownScoping(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, _ := newTestStore()
|
||||
cands := []Candidate{candidate("ns/p1", 5, 0)}
|
||||
|
||||
l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
|
||||
if err := s.Report(context.Background(), l.ID, ResultRateLimited, "example.com"); err != nil {
|
||||
t.Fatalf("Report: %v", err)
|
||||
}
|
||||
|
||||
// Same target: excluded.
|
||||
_, stats, err := s.Acquire(context.Background(), AcquireRequest{
|
||||
Candidates: cands, Target: "example.com", TTL: time.Minute,
|
||||
})
|
||||
if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
|
||||
t.Errorf("same-target acquire = (%v, %+v), want ErrNoMatch with InCooldown=1", err, stats)
|
||||
}
|
||||
|
||||
// Different target: fine.
|
||||
mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "other.org", TTL: time.Minute})
|
||||
|
||||
// No target (global pool): a target-scoped cooldown does not apply.
|
||||
mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
|
||||
}
|
||||
|
||||
func TestAcquire_globalCooldownBlocksEverything(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, _ := newTestStore()
|
||||
cands := []Candidate{candidate("ns/p1", 5, 0)}
|
||||
|
||||
// A lease without a target, reported banned without a target: the
|
||||
// cooldown lands on the global pool.
|
||||
l := mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
|
||||
if err := s.Report(context.Background(), l.ID, ResultBanned, ""); err != nil {
|
||||
t.Fatalf("Report: %v", err)
|
||||
}
|
||||
|
||||
for _, target := range []string{"", "example.com"} {
|
||||
_, stats, err := s.Acquire(context.Background(), AcquireRequest{
|
||||
Candidates: cands, Target: target, TTL: time.Minute,
|
||||
})
|
||||
if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
|
||||
t.Errorf("acquire(target=%q) = (%v, %+v), want global cooldown to block", target, err, stats)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquire_cooldownExpires(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, clock := newTestStore()
|
||||
cands := []Candidate{candidate("ns/p1", 5, 0)}
|
||||
|
||||
l := mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
|
||||
if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); err != nil {
|
||||
t.Fatalf("Report: %v", err)
|
||||
}
|
||||
if _, _, err := s.Acquire(context.Background(), AcquireRequest{Candidates: cands, TTL: time.Minute}); !errors.Is(err, ErrNoMatch) {
|
||||
t.Fatal("expected cooldown to block immediately after the report")
|
||||
}
|
||||
|
||||
clock.Advance(15*time.Minute + time.Second)
|
||||
mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
|
||||
}
|
||||
|
||||
func TestExpiry_freesCapacityWithoutSweep(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, clock := newTestStore()
|
||||
req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 1, 0)}, TTL: time.Minute}
|
||||
|
||||
mustAcquire(t, s, req)
|
||||
if _, _, err := s.Acquire(context.Background(), req); !errors.Is(err, ErrNoMatch) {
|
||||
t.Fatal("capacity 1 not enforced")
|
||||
}
|
||||
|
||||
clock.Advance(2 * time.Minute)
|
||||
// No sweep has run; expiry must still free capacity and zero the counts.
|
||||
if got := s.ActiveCount("ns/p1"); got != 0 {
|
||||
t.Fatalf("ActiveCount after TTL = %d, want 0", got)
|
||||
}
|
||||
if counts := s.Counts(); len(counts) != 0 {
|
||||
t.Fatalf("Counts after TTL = %v, want empty", counts)
|
||||
}
|
||||
mustAcquire(t, s, req)
|
||||
}
|
||||
|
||||
func TestReport_expiredButRetainedLease(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, clock := newTestStore()
|
||||
cands := []Candidate{candidate("ns/p1", 5, 0)}
|
||||
|
||||
l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
|
||||
|
||||
// TTL lapses; the report arrives late — exactly when the proxy is being
|
||||
// rate-limited, which is when the cooldown matters most.
|
||||
clock.Advance(5 * time.Minute)
|
||||
s.sweep(clock.Now())
|
||||
if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); err != nil {
|
||||
t.Fatalf("Report on an expired-but-retained lease: %v", err)
|
||||
}
|
||||
// The cooldown fell back to the lease's own target.
|
||||
_, stats, err := s.Acquire(context.Background(), AcquireRequest{
|
||||
Candidates: cands, Target: "example.com", TTL: time.Minute,
|
||||
})
|
||||
if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
|
||||
t.Errorf("acquire = (%v, %+v), want cooldown from the late report", err, stats)
|
||||
}
|
||||
|
||||
// Past the retention window the sweep finally drops it.
|
||||
clock.Advance(15 * time.Minute)
|
||||
s.sweep(clock.Now())
|
||||
if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); !errors.Is(err, ErrUnknownLease) {
|
||||
t.Errorf("Report after retention = %v, want ErrUnknownLease", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReport_okRecordsNothing(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, _ := newTestStore()
|
||||
cands := []Candidate{candidate("ns/p1", 5, 0)}
|
||||
l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
|
||||
|
||||
if err := s.Report(context.Background(), l.ID, ResultOK, "example.com"); err != nil {
|
||||
t.Fatalf("Report(ok): %v", err)
|
||||
}
|
||||
mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
|
||||
}
|
||||
|
||||
func TestRelease_isIdempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, _ := newTestStore()
|
||||
l := mustAcquire(t, s, AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 1, 0)}, TTL: time.Minute})
|
||||
|
||||
s.Release(context.Background(), l.ID)
|
||||
s.Release(context.Background(), l.ID)
|
||||
s.Release(context.Background(), "never-existed")
|
||||
if got := s.ActiveCount("ns/p1"); got != 0 {
|
||||
t.Errorf("ActiveCount = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseResult(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, valid := range []string{"ok", "rate_limited", "banned"} {
|
||||
if _, ok := ParseResult(valid); !ok {
|
||||
t.Errorf("ParseResult(%q) rejected a valid value", valid)
|
||||
}
|
||||
}
|
||||
for _, invalid := range []string{"", "OK", "throttled", "rate-limited"} {
|
||||
if _, ok := ParseResult(invalid); ok {
|
||||
t.Errorf("ParseResult(%q) accepted an invalid value", invalid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquire_concurrentNeverOvercommits(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, _ := newTestStore()
|
||||
req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 5, 0)}, TTL: time.Minute}
|
||||
|
||||
const attempts = 40
|
||||
var wg sync.WaitGroup
|
||||
granted := make(chan *Lease, attempts)
|
||||
for range attempts {
|
||||
wg.Go(func() {
|
||||
if l, _, err := s.Acquire(context.Background(), req); err == nil {
|
||||
granted <- l
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
close(granted)
|
||||
|
||||
var n int
|
||||
for range granted {
|
||||
n++
|
||||
}
|
||||
if n != 5 {
|
||||
t.Errorf("%d of %d concurrent acquires granted, want exactly MaxLeases=5", n, attempts)
|
||||
}
|
||||
if got := s.ActiveCount("ns/p1"); got != 5 {
|
||||
t.Errorf("ActiveCount = %d, want 5", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStart_sweepsAndStops(t *testing.T) {
|
||||
t.Parallel()
|
||||
s, clock := newTestStore()
|
||||
s.SweepInterval = time.Millisecond
|
||||
|
||||
l := mustAcquire(t, s, AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 5, 0)}, TTL: time.Minute})
|
||||
clock.Advance(20 * time.Minute) // past TTL + retention
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s.Start(ctx) }()
|
||||
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
if err := s.Report(context.Background(), l.ID, ResultOK, ""); errors.Is(err, ErrUnknownLease) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("sweep never dropped the lease")
|
||||
case <-time.After(5 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Errorf("Start returned %v, want nil", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Start did not stop on cancel")
|
||||
}
|
||||
}
|
||||
119
internal/metrics/metrics.go
Normal file
119
internal/metrics/metrics.go
Normal file
@@ -0,0 +1,119 @@
|
||||
// Package metrics defines the operator's Prometheus metrics. Nothing here
|
||||
// registers itself — no init(), per house rules — the composition root
|
||||
// calls Register explicitly, which also lets every test use a fresh
|
||||
// registry.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// Metrics holds the vector metrics the operator's components feed. The
|
||||
// consuming packages (health, discovery, provider) each define their own
|
||||
// small recorder interface; *Metrics satisfies all of them structurally,
|
||||
// so none of them import this package's prometheus surface.
|
||||
type Metrics struct {
|
||||
healthcheckDuration *prometheus.HistogramVec
|
||||
healthcheckFailures *prometheus.CounterVec
|
||||
leaseRequests *prometheus.CounterVec
|
||||
providerRequests *prometheus.CounterVec
|
||||
}
|
||||
|
||||
// New builds the metric set, unregistered.
|
||||
func New() *Metrics {
|
||||
return &Metrics{
|
||||
healthcheckDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "proxy_operator_healthcheck_duration_seconds",
|
||||
Help: "Duration of through-the-proxy health probes.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{"proxy"}),
|
||||
healthcheckFailures: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "proxy_operator_healthcheck_failures_total",
|
||||
Help: "Failed health probes.",
|
||||
}, []string{"proxy"}),
|
||||
leaseRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "proxy_operator_lease_requests_total",
|
||||
Help: "Lease acquisition requests by outcome.",
|
||||
}, []string{"outcome"}),
|
||||
providerRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "proxy_operator_provider_requests_total",
|
||||
Help: "Provider API calls by operation and classified result.",
|
||||
}, []string{"provider", "op", "result"}),
|
||||
}
|
||||
}
|
||||
|
||||
// Register registers the vectors plus the two scrape-time collectors.
|
||||
// proxyPhases and activeLeases are read at every scrape: gauges derived
|
||||
// from reconcile-time increments inevitably drift and leak series on
|
||||
// delete; reading the source of truth cannot.
|
||||
func (m *Metrics) Register(reg prometheus.Registerer, proxyPhases func() map[string]int, activeLeases func() int) error {
|
||||
collectors := []prometheus.Collector{
|
||||
m.healthcheckDuration,
|
||||
m.healthcheckFailures,
|
||||
m.leaseRequests,
|
||||
m.providerRequests,
|
||||
&constCollector{
|
||||
desc: prometheus.NewDesc("proxy_operator_proxies",
|
||||
"Proxy objects by phase.", []string{"phase"}, nil),
|
||||
read: proxyPhases,
|
||||
},
|
||||
&constCollector{
|
||||
desc: prometheus.NewDesc("proxy_operator_leases_active",
|
||||
"Currently active leases.", nil, nil),
|
||||
read: func() map[string]int { return map[string]int{"": activeLeases()} },
|
||||
},
|
||||
}
|
||||
for _, c := range collectors {
|
||||
if err := reg.Register(c); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ObserveProbe records one health probe. Called on every probe — metrics
|
||||
// are the home for high-frequency signal that must never touch status.
|
||||
func (m *Metrics) ObserveProbe(proxy string, latency time.Duration, success bool) {
|
||||
m.healthcheckDuration.WithLabelValues(proxy).Observe(latency.Seconds())
|
||||
if !success {
|
||||
m.healthcheckFailures.WithLabelValues(proxy).Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// ForgetProxy drops the per-proxy series when the health engine prunes its
|
||||
// state — without this, series for deleted proxies leak forever.
|
||||
func (m *Metrics) ForgetProxy(proxy string) {
|
||||
m.healthcheckDuration.DeleteLabelValues(proxy)
|
||||
m.healthcheckFailures.DeleteLabelValues(proxy)
|
||||
}
|
||||
|
||||
// LeaseRequest records a lease acquisition outcome ("granted"|"no_match").
|
||||
func (m *Metrics) LeaseRequest(outcome string) {
|
||||
m.leaseRequests.WithLabelValues(outcome).Inc()
|
||||
}
|
||||
|
||||
// ProviderRequest records one provider API call with its classified result.
|
||||
func (m *Metrics) ProviderRequest(provider, op, result string) {
|
||||
m.providerRequests.WithLabelValues(provider, op, result).Inc()
|
||||
}
|
||||
|
||||
// constCollector reads a label→value map at scrape time and emits one
|
||||
// gauge sample per entry. An empty-string label key means "no labels".
|
||||
type constCollector struct {
|
||||
desc *prometheus.Desc
|
||||
read func() map[string]int
|
||||
}
|
||||
|
||||
func (c *constCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.desc }
|
||||
|
||||
func (c *constCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
for label, value := range c.read() {
|
||||
if label == "" {
|
||||
ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(value))
|
||||
continue
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(value), label)
|
||||
}
|
||||
}
|
||||
113
internal/metrics/metrics_test.go
Normal file
113
internal/metrics/metrics_test.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
)
|
||||
|
||||
// register wires a fresh registry — the reason Register exists instead of
|
||||
// init()-time self-registration.
|
||||
func register(t *testing.T, m *Metrics, phases map[string]int, active int) *prometheus.Registry {
|
||||
t.Helper()
|
||||
reg := prometheus.NewRegistry()
|
||||
err := m.Register(reg,
|
||||
func() map[string]int { return phases },
|
||||
func() int { return active },
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Register: %v", err)
|
||||
}
|
||||
return reg
|
||||
}
|
||||
|
||||
func TestRegister_scrapeTimeCollectors(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := New()
|
||||
reg := register(t, m, map[string]int{"Ready": 3, "Provisioning": 1}, 7)
|
||||
|
||||
expected := `
|
||||
# HELP proxy_operator_leases_active Currently active leases.
|
||||
# TYPE proxy_operator_leases_active gauge
|
||||
proxy_operator_leases_active 7
|
||||
# HELP proxy_operator_proxies Proxy objects by phase.
|
||||
# TYPE proxy_operator_proxies gauge
|
||||
proxy_operator_proxies{phase="Provisioning"} 1
|
||||
proxy_operator_proxies{phase="Ready"} 3
|
||||
`
|
||||
if err := testutil.GatherAndCompare(reg, strings.NewReader(expected),
|
||||
"proxy_operator_proxies", "proxy_operator_leases_active"); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserveProbe_andForget(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := New()
|
||||
reg := register(t, m, nil, 0)
|
||||
|
||||
m.ObserveProbe("default/p1", 30*time.Millisecond, true)
|
||||
m.ObserveProbe("default/p1", 40*time.Millisecond, false)
|
||||
m.ObserveProbe("default/p2", 10*time.Millisecond, true)
|
||||
|
||||
if got := testutil.CollectAndCount(m.healthcheckDuration); got != 2 {
|
||||
t.Errorf("duration series = %d, want 2 (one per proxy)", got)
|
||||
}
|
||||
if got := testutil.ToFloat64(m.healthcheckFailures.WithLabelValues("default/p1")); got != 1 {
|
||||
t.Errorf("p1 failures = %v, want 1 (only the failed probe)", got)
|
||||
}
|
||||
|
||||
m.ForgetProxy("default/p1")
|
||||
if got := testutil.CollectAndCount(m.healthcheckDuration); got != 1 {
|
||||
t.Errorf("duration series after ForgetProxy = %d, want 1 — series must not leak", got)
|
||||
}
|
||||
if got := testutil.CollectAndCount(m.healthcheckFailures); got != 0 {
|
||||
t.Errorf("failure series after ForgetProxy = %d, want 0", got)
|
||||
}
|
||||
_ = reg
|
||||
}
|
||||
|
||||
func TestLeaseRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := New()
|
||||
register(t, m, nil, 0)
|
||||
|
||||
m.LeaseRequest("granted")
|
||||
m.LeaseRequest("granted")
|
||||
m.LeaseRequest("no_match")
|
||||
|
||||
if got := testutil.ToFloat64(m.leaseRequests.WithLabelValues("granted")); got != 2 {
|
||||
t.Errorf("granted = %v, want 2", got)
|
||||
}
|
||||
if got := testutil.ToFloat64(m.leaseRequests.WithLabelValues("no_match")); got != 1 {
|
||||
t.Errorf("no_match = %v, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := New()
|
||||
register(t, m, nil, 0)
|
||||
|
||||
m.ProviderRequest("gcp-eu", "create", "ok")
|
||||
m.ProviderRequest("gcp-eu", "create", "quota_exceeded")
|
||||
|
||||
if got := testutil.ToFloat64(m.providerRequests.WithLabelValues("gcp-eu", "create", "ok")); got != 1 {
|
||||
t.Errorf("ok = %v, want 1", got)
|
||||
}
|
||||
if got := testutil.ToFloat64(m.providerRequests.WithLabelValues("gcp-eu", "create", "quota_exceeded")); got != 1 {
|
||||
t.Errorf("quota_exceeded = %v, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegister_freshRegistryPerTest(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Registering the same metric set on two registries must both succeed —
|
||||
// the property init()-style global registration would break.
|
||||
m1, m2 := New(), New()
|
||||
register(t, m1, nil, 0)
|
||||
register(t, m2, nil, 0)
|
||||
}
|
||||
61
internal/provider/gcp/errors.go
Normal file
61
internal/provider/gcp/errors.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package gcp
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"slices"
|
||||
|
||||
"google.golang.org/api/googleapi"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// classify maps a GCP API error onto the provider taxonomy:
|
||||
// 404 → NotFound; 429 and quota-flavored 403s → QuotaExceeded;
|
||||
// 400/401/other 403s → Permanent; 408/5xx and anything unrecognized
|
||||
// (network errors, context cancellation) → Transient, because retrying is
|
||||
// always safer than latching Failed on an error nobody taught this
|
||||
// function to recognize.
|
||||
func classify(err error) error {
|
||||
var gerr *googleapi.Error
|
||||
if !errors.As(err, &gerr) {
|
||||
return provider.ErrTransient
|
||||
}
|
||||
switch {
|
||||
case gerr.Code == http.StatusNotFound:
|
||||
return provider.ErrNotFound
|
||||
case gerr.Code == http.StatusTooManyRequests:
|
||||
return provider.ErrQuotaExceeded
|
||||
case gerr.Code == http.StatusForbidden && hasReason(gerr, "quotaExceeded", "rateLimitExceeded"):
|
||||
return provider.ErrQuotaExceeded
|
||||
case gerr.Code == http.StatusBadRequest,
|
||||
gerr.Code == http.StatusUnauthorized,
|
||||
gerr.Code == http.StatusForbidden:
|
||||
return provider.ErrPermanent
|
||||
default:
|
||||
return provider.ErrTransient
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) wrapErr(op, id string, err error) error {
|
||||
return provider.Wrap(classify(err), op, p.name, id, err)
|
||||
}
|
||||
|
||||
func hasReason(gerr *googleapi.Error, reasons ...string) bool {
|
||||
for _, item := range gerr.Errors {
|
||||
if slices.Contains(reasons, item.Reason) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isAlreadyExists(err error) bool {
|
||||
var gerr *googleapi.Error
|
||||
return errors.As(err, &gerr) && gerr.Code == http.StatusConflict
|
||||
}
|
||||
|
||||
func isNotFound(err error) bool {
|
||||
var gerr *googleapi.Error
|
||||
return errors.As(err, &gerr) && gerr.Code == http.StatusNotFound
|
||||
}
|
||||
71
internal/provider/gcp/errors_test.go
Normal file
71
internal/provider/gcp/errors_test.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package gcp
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"google.golang.org/api/googleapi"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
func gerr(code int, reasons ...string) error {
|
||||
e := &googleapi.Error{Code: code, Message: "boom"}
|
||||
for _, r := range reasons {
|
||||
e.Errors = append(e.Errors, googleapi.ErrorItem{Reason: r})
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want error
|
||||
}{
|
||||
{name: "404 is NotFound", err: gerr(404), want: provider.ErrNotFound},
|
||||
{name: "429 is Quota", err: gerr(429), want: provider.ErrQuotaExceeded},
|
||||
{name: "403 quotaExceeded is Quota", err: gerr(403, "quotaExceeded"), want: provider.ErrQuotaExceeded},
|
||||
{name: "403 rateLimitExceeded is Quota", err: gerr(403, "rateLimitExceeded"), want: provider.ErrQuotaExceeded},
|
||||
{name: "403 plain is Permanent", err: gerr(403, "forbidden"), want: provider.ErrPermanent},
|
||||
{name: "400 is Permanent", err: gerr(400), want: provider.ErrPermanent},
|
||||
{name: "401 is Permanent", err: gerr(401), want: provider.ErrPermanent},
|
||||
{name: "408 is Transient", err: gerr(408), want: provider.ErrTransient},
|
||||
{name: "500 is Transient", err: gerr(500), want: provider.ErrTransient},
|
||||
{name: "503 is Transient", err: gerr(503), want: provider.ErrTransient},
|
||||
{name: "409 is Transient (alreadyExists is handled before classify)", err: gerr(409), want: provider.ErrTransient},
|
||||
{name: "plain network error is Transient", err: errors.New("connection reset"), want: provider.ErrTransient},
|
||||
{name: "wrapped googleapi error still classifies", err: fmt.Errorf("calling api: %w", gerr(404)), want: provider.ErrNotFound},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := classify(tc.err); got != tc.want {
|
||||
t.Errorf("classify() = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The wrapped error must satisfy both halves of the taxonomy contract:
|
||||
// errors.Is against the sentinel AND errors.As back to the SDK error.
|
||||
func TestWrapErr_isAndAsBothWork(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := &Provider{name: "gcp-eu"}
|
||||
wrapped := p.wrapErr("get", "zones/z/instances/i", gerr(404))
|
||||
|
||||
if !errors.Is(wrapped, provider.ErrNotFound) {
|
||||
t.Error("errors.Is(wrapped, ErrNotFound) = false")
|
||||
}
|
||||
var ge *googleapi.Error
|
||||
if !errors.As(wrapped, &ge) || ge.Code != 404 {
|
||||
t.Error("errors.As back to *googleapi.Error failed")
|
||||
}
|
||||
if provider.Class(wrapped) != provider.ErrNotFound {
|
||||
t.Errorf("Class() = %v, want ErrNotFound", provider.Class(wrapped))
|
||||
}
|
||||
}
|
||||
240
internal/provider/gcp/gcp.go
Normal file
240
internal/provider/gcp/gcp.go
Normal file
@@ -0,0 +1,240 @@
|
||||
// Package gcp implements the provider contract on GCP Compute Engine via
|
||||
// the modern Cloud Client Library (cloud.google.com/go/compute/apiv1),
|
||||
// deliberately restricted to four calls: instances.Insert, Get, Delete,
|
||||
// AggregatedList. Operations are fire-and-forget — Operation.Wait is never
|
||||
// called; Create/Delete return as soon as the operation is submitted and
|
||||
// the reconciler discovers progress by polling Get.
|
||||
package gcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
compute "cloud.google.com/go/compute/apiv1"
|
||||
"cloud.google.com/go/compute/apiv1/computepb"
|
||||
"google.golang.org/api/iterator"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// instancesAPI is the test seam. It deliberately does not mirror the SDK:
|
||||
// the SDK's InstancesScopedListPairIterator has an unexported nextFunc, so
|
||||
// a fake cannot construct one — the seam flattens AggregatedList to a
|
||||
// slice, and returns operations as just their name (the only thing this
|
||||
// provider ever uses, since it never waits on them).
|
||||
type instancesAPI interface {
|
||||
Insert(ctx context.Context, req *computepb.InsertInstanceRequest) (opName string, err error)
|
||||
Get(ctx context.Context, req *computepb.GetInstanceRequest) (*computepb.Instance, error)
|
||||
Delete(ctx context.Context, req *computepb.DeleteInstanceRequest) (opName string, err error)
|
||||
AggregatedList(ctx context.Context, req *computepb.AggregatedListInstancesRequest) ([]*computepb.Instance, error)
|
||||
}
|
||||
|
||||
// realInstances adapts *compute.InstancesClient to the seam.
|
||||
type realInstances struct {
|
||||
client *compute.InstancesClient
|
||||
}
|
||||
|
||||
func (r *realInstances) Insert(ctx context.Context, req *computepb.InsertInstanceRequest) (string, error) {
|
||||
op, err := r.client.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return op.Name(), nil
|
||||
}
|
||||
|
||||
func (r *realInstances) Get(ctx context.Context, req *computepb.GetInstanceRequest) (*computepb.Instance, error) {
|
||||
return r.client.Get(ctx, req)
|
||||
}
|
||||
|
||||
func (r *realInstances) Delete(ctx context.Context, req *computepb.DeleteInstanceRequest) (string, error) {
|
||||
op, err := r.client.Delete(ctx, req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return op.Name(), nil
|
||||
}
|
||||
|
||||
func (r *realInstances) AggregatedList(ctx context.Context, req *computepb.AggregatedListInstancesRequest) ([]*computepb.Instance, error) {
|
||||
it := r.client.AggregatedList(ctx, req)
|
||||
var out []*computepb.Instance
|
||||
for {
|
||||
pair, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
return out, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if pair.Value != nil {
|
||||
out = append(out, pair.Value.Instances...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Provider implements provider.Provider on GCP Compute Engine.
|
||||
type Provider struct {
|
||||
name string
|
||||
cfg provider.GCPConfig
|
||||
api instancesAPI
|
||||
}
|
||||
|
||||
// New builds a Provider using Application Default Credentials (workload
|
||||
// identity in-cluster, gcloud ADC locally — no key-file plumbing).
|
||||
// Deliberately untested: it dials real Google endpoints; everything below
|
||||
// it is exercised through newWithAPI.
|
||||
func New(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
|
||||
client, err := compute.NewInstancesRESTClient(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating GCP instances client: %w", err)
|
||||
}
|
||||
return newWithAPI(pc, &realInstances{client: client}), nil
|
||||
}
|
||||
|
||||
func newWithAPI(pc provider.ProviderConfig, api instancesAPI) *Provider {
|
||||
cfg := provider.GCPConfig{}
|
||||
if pc.GCP != nil {
|
||||
cfg = *pc.GCP
|
||||
}
|
||||
return &Provider{name: pc.Name, cfg: withDefaults(cfg), api: api}
|
||||
}
|
||||
|
||||
// Create submits the insert and returns immediately with the
|
||||
// zone-qualified providerID. A 409 alreadyExists is success — the
|
||||
// deterministic instance name means a repeat call after a crash found the
|
||||
// VM it already created, which is exactly the idempotency the contract
|
||||
// demands.
|
||||
func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
|
||||
pl := req.Placement
|
||||
if pl.Zone == "" || pl.MachineType == "" || pl.Image == "" {
|
||||
return "", provider.Wrap(provider.ErrPermanent, "create", p.name, "", fmt.Errorf(
|
||||
"gcp requires placement.zone, placement.machineType and placement.image (got zone=%q machineType=%q image=%q)",
|
||||
pl.Zone, pl.MachineType, pl.Image))
|
||||
}
|
||||
id := formatProviderID(pl.Zone, req.Name)
|
||||
if _, err := p.api.Insert(ctx, buildInsertRequest(p.cfg, req)); err != nil && !isAlreadyExists(err) {
|
||||
return "", p.wrapErr("create", id, err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// Get returns the instance state. The providerID carries its own zone, so
|
||||
// this stays correct even mid-replacement after a zone edit — re-reading
|
||||
// spec.placement.zone would look up the wrong zone exactly then.
|
||||
func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instance, error) {
|
||||
zone, name, err := parseProviderID(providerID)
|
||||
if err != nil {
|
||||
return nil, provider.Wrap(provider.ErrPermanent, "get", p.name, providerID, err)
|
||||
}
|
||||
inst, err := p.api.Get(ctx, &computepb.GetInstanceRequest{
|
||||
Project: p.cfg.Project,
|
||||
Zone: zone,
|
||||
Instance: name,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, p.wrapErr("get", providerID, err)
|
||||
}
|
||||
return toInstance(inst, zone), nil
|
||||
}
|
||||
|
||||
// Delete submits the delete and returns; deleting an instance that is
|
||||
// already gone is success.
|
||||
func (p *Provider) Delete(ctx context.Context, providerID string) error {
|
||||
zone, name, err := parseProviderID(providerID)
|
||||
if err != nil {
|
||||
return provider.Wrap(provider.ErrPermanent, "delete", p.name, providerID, err)
|
||||
}
|
||||
if _, err := p.api.Delete(ctx, &computepb.DeleteInstanceRequest{
|
||||
Project: p.cfg.Project,
|
||||
Zone: zone,
|
||||
Instance: name,
|
||||
}); err != nil && !isNotFound(err) {
|
||||
return p.wrapErr("delete", providerID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListByTag sweeps every zone for instances carrying the GC labels.
|
||||
// ReturnPartialSuccess matters: without it one unreachable zone fails the
|
||||
// entire GC sweep.
|
||||
func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
|
||||
instances, err := p.api.AggregatedList(ctx, &computepb.AggregatedListInstancesRequest{
|
||||
Project: p.cfg.Project,
|
||||
Filter: proto.String(fmt.Sprintf("labels.%s = %s", provider.LabelManaged, provider.LabelManagedYes)),
|
||||
ReturnPartialSuccess: proto.Bool(true),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, p.wrapErr("list", "", err)
|
||||
}
|
||||
out := make([]provider.Instance, 0, len(instances))
|
||||
for _, inst := range instances {
|
||||
out = append(out, *toInstance(inst, lastPathSegment(inst.GetZone())))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func toInstance(inst *computepb.Instance, zone string) *provider.Instance {
|
||||
var ip string
|
||||
if nics := inst.GetNetworkInterfaces(); len(nics) > 0 {
|
||||
if acs := nics[0].GetAccessConfigs(); len(acs) > 0 {
|
||||
ip = acs[0].GetNatIP()
|
||||
}
|
||||
}
|
||||
// CreationTimestamp is RFC3339; a parse failure leaves the zero time,
|
||||
// which orphan GC treats as "old" — safe, since a malformed timestamp
|
||||
// never protects a candidate from collection forever.
|
||||
created, _ := time.Parse(time.RFC3339, inst.GetCreationTimestamp())
|
||||
return &provider.Instance{
|
||||
ID: formatProviderID(zone, inst.GetName()),
|
||||
IP: ip,
|
||||
State: mapState(inst.GetStatus(), ip),
|
||||
UID: inst.GetLabels()[provider.LabelUID],
|
||||
CreatedAt: created,
|
||||
}
|
||||
}
|
||||
|
||||
// mapState collapses GCP instance statuses onto the provider states. A
|
||||
// RUNNING instance without a NatIP maps to Provisioning — an empty IP must
|
||||
// never be published as Running. Anything unrecognized maps to Stopped:
|
||||
// the reconciler's answer to Stopped is delete-and-recreate, which is
|
||||
// always safe for cattle.
|
||||
func mapState(status, ip string) provider.InstanceState {
|
||||
switch status {
|
||||
case "PROVISIONING", "STAGING", "REPAIRING":
|
||||
return provider.StateProvisioning
|
||||
case "RUNNING":
|
||||
if ip == "" {
|
||||
return provider.StateProvisioning
|
||||
}
|
||||
return provider.StateRunning
|
||||
case "STOPPING", "STOPPED", "SUSPENDING", "SUSPENDED":
|
||||
return provider.StateStopped
|
||||
case "TERMINATED":
|
||||
return provider.StateTerminated
|
||||
default:
|
||||
return provider.StateStopped
|
||||
}
|
||||
}
|
||||
|
||||
func formatProviderID(zone, name string) string {
|
||||
return fmt.Sprintf("zones/%s/instances/%s", zone, name)
|
||||
}
|
||||
|
||||
func parseProviderID(id string) (zone, name string, err error) {
|
||||
parts := strings.Split(id, "/")
|
||||
if len(parts) != 4 || parts[0] != "zones" || parts[2] != "instances" || parts[1] == "" || parts[3] == "" {
|
||||
return "", "", fmt.Errorf("malformed gcp providerID %q, want zones/<zone>/instances/<name>", id)
|
||||
}
|
||||
return parts[1], parts[3], nil
|
||||
}
|
||||
|
||||
// lastPathSegment extracts the zone name from the URL-style
|
||||
// ".../zones/europe-west1-b" the API returns on instances.
|
||||
func lastPathSegment(url string) string {
|
||||
if i := strings.LastIndexByte(url, '/'); i >= 0 {
|
||||
return url[i+1:]
|
||||
}
|
||||
return url
|
||||
}
|
||||
271
internal/provider/gcp/gcp_test.go
Normal file
271
internal/provider/gcp/gcp_test.go
Normal file
@@ -0,0 +1,271 @@
|
||||
package gcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/compute/apiv1/computepb"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// fakeAPI implements the instancesAPI seam.
|
||||
type fakeAPI struct {
|
||||
insertReq *computepb.InsertInstanceRequest
|
||||
insertErr error
|
||||
|
||||
getReq *computepb.GetInstanceRequest
|
||||
getInst *computepb.Instance
|
||||
getErr error
|
||||
|
||||
deleteReq *computepb.DeleteInstanceRequest
|
||||
deleteErr error
|
||||
|
||||
listReq *computepb.AggregatedListInstancesRequest
|
||||
listInsts []*computepb.Instance
|
||||
listErr error
|
||||
}
|
||||
|
||||
func (f *fakeAPI) Insert(_ context.Context, req *computepb.InsertInstanceRequest) (string, error) {
|
||||
f.insertReq = req
|
||||
return "op-insert", f.insertErr
|
||||
}
|
||||
|
||||
func (f *fakeAPI) Get(_ context.Context, req *computepb.GetInstanceRequest) (*computepb.Instance, error) {
|
||||
f.getReq = req
|
||||
return f.getInst, f.getErr
|
||||
}
|
||||
|
||||
func (f *fakeAPI) Delete(_ context.Context, req *computepb.DeleteInstanceRequest) (string, error) {
|
||||
f.deleteReq = req
|
||||
return "op-delete", f.deleteErr
|
||||
}
|
||||
|
||||
func (f *fakeAPI) AggregatedList(_ context.Context, req *computepb.AggregatedListInstancesRequest) ([]*computepb.Instance, error) {
|
||||
f.listReq = req
|
||||
return f.listInsts, f.listErr
|
||||
}
|
||||
|
||||
func newTestProvider(api *fakeAPI) *Provider {
|
||||
return newWithAPI(provider.ProviderConfig{
|
||||
Name: "gcp-eu",
|
||||
Type: "gcp",
|
||||
GCP: &provider.GCPConfig{Project: "my-project"},
|
||||
}, api)
|
||||
}
|
||||
|
||||
func TestCreate_returnsZoneQualifiedID(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := &fakeAPI{}
|
||||
p := newTestProvider(api)
|
||||
|
||||
id, err := p.Create(context.Background(), testCreateRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
if want := "zones/europe-west1-b/instances/proxy-abc123def456ghij"; id != want {
|
||||
t.Errorf("providerID = %s, want %s", id, want)
|
||||
}
|
||||
if api.insertReq.Project != "my-project" || api.insertReq.Zone != "europe-west1-b" {
|
||||
t.Errorf("insert sent to %s/%s, want my-project/europe-west1-b", api.insertReq.Project, api.insertReq.Zone)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_alreadyExistsIsSuccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := &fakeAPI{insertErr: gerr(409)}
|
||||
p := newTestProvider(api)
|
||||
|
||||
id, err := p.Create(context.Background(), testCreateRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Create after crash (409): %v — alreadyExists must be success", err)
|
||||
}
|
||||
if want := "zones/europe-west1-b/instances/proxy-abc123def456ghij"; id != want {
|
||||
t.Errorf("providerID = %s, want %s", id, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_incompletePlacementIsPermanent(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := &fakeAPI{}
|
||||
p := newTestProvider(api)
|
||||
req := testCreateRequest()
|
||||
req.Placement.MachineType = ""
|
||||
|
||||
_, err := p.Create(context.Background(), req)
|
||||
if provider.Class(err) != provider.ErrPermanent {
|
||||
t.Errorf("Class = %v, want ErrPermanent for missing placement", provider.Class(err))
|
||||
}
|
||||
if api.insertReq != nil {
|
||||
t.Error("Insert was called despite invalid placement")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_quotaErrorClassified(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := newTestProvider(&fakeAPI{insertErr: gerr(403, "quotaExceeded")})
|
||||
_, err := p.Create(context.Background(), testCreateRequest())
|
||||
if provider.Class(err) != provider.ErrQuotaExceeded {
|
||||
t.Errorf("Class = %v, want ErrQuotaExceeded", provider.Class(err))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_stateMapping(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
status string
|
||||
natIP string
|
||||
wantState provider.InstanceState
|
||||
wantIP string
|
||||
}{
|
||||
{name: "provisioning", status: "PROVISIONING", wantState: provider.StateProvisioning},
|
||||
{name: "staging", status: "STAGING", wantState: provider.StateProvisioning},
|
||||
{name: "repairing", status: "REPAIRING", wantState: provider.StateProvisioning},
|
||||
{name: "running without NatIP stays provisioning", status: "RUNNING", wantState: provider.StateProvisioning},
|
||||
{name: "running with NatIP", status: "RUNNING", natIP: "34.1.2.3", wantState: provider.StateRunning, wantIP: "34.1.2.3"},
|
||||
{name: "stopped", status: "STOPPED", wantState: provider.StateStopped},
|
||||
{name: "suspended", status: "SUSPENDED", wantState: provider.StateStopped},
|
||||
{name: "terminated", status: "TERMINATED", wantState: provider.StateTerminated},
|
||||
{name: "unknown status maps to stopped for recreation", status: "SOMETHING_NEW", wantState: provider.StateStopped},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
inst := &computepb.Instance{
|
||||
Name: proto.String("proxy-abc"),
|
||||
Status: proto.String(tc.status),
|
||||
CreationTimestamp: proto.String("2026-08-09T10:00:00+02:00"),
|
||||
Labels: map[string]string{
|
||||
provider.LabelUID: "uid-1",
|
||||
},
|
||||
}
|
||||
if tc.natIP != "" {
|
||||
inst.NetworkInterfaces = []*computepb.NetworkInterface{{
|
||||
AccessConfigs: []*computepb.AccessConfig{{NatIP: proto.String(tc.natIP)}},
|
||||
}}
|
||||
}
|
||||
p := newTestProvider(&fakeAPI{getInst: inst})
|
||||
|
||||
got, err := p.Get(context.Background(), "zones/europe-west1-b/instances/proxy-abc")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if got.State != tc.wantState || got.IP != tc.wantIP {
|
||||
t.Errorf("state/ip = %s/%q, want %s/%q", got.State, got.IP, tc.wantState, tc.wantIP)
|
||||
}
|
||||
if got.UID != "uid-1" {
|
||||
t.Errorf("UID = %q, want uid-1 (from the GC label)", got.UID)
|
||||
}
|
||||
if got.ID != "zones/europe-west1-b/instances/proxy-abc" {
|
||||
t.Errorf("ID = %s, want the zone-qualified providerID", got.ID)
|
||||
}
|
||||
if got.CreatedAt.IsZero() {
|
||||
t.Error("CreatedAt not parsed from creationTimestamp")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_notFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := newTestProvider(&fakeAPI{getErr: gerr(404)})
|
||||
_, err := p.Get(context.Background(), "zones/z/instances/gone")
|
||||
if !errors.Is(err, provider.ErrNotFound) {
|
||||
t.Errorf("err = %v, want ErrNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_malformedProviderID(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := newTestProvider(&fakeAPI{})
|
||||
for _, id := range []string{"", "proxy-abc", "zones//instances/x", "zones/z/instances/", "z/zone/i/name"} {
|
||||
if _, err := p.Get(context.Background(), id); provider.Class(err) != provider.ErrPermanent {
|
||||
t.Errorf("Get(%q): Class = %v, want ErrPermanent", id, provider.Class(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_notFoundIsSuccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := &fakeAPI{deleteErr: gerr(404)}
|
||||
p := newTestProvider(api)
|
||||
if err := p.Delete(context.Background(), "zones/z/instances/gone"); err != nil {
|
||||
t.Errorf("Delete of missing instance: %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_sendsParsedZoneAndName(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := &fakeAPI{}
|
||||
p := newTestProvider(api)
|
||||
if err := p.Delete(context.Background(), "zones/us-east1-c/instances/proxy-xyz"); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
if api.deleteReq.Zone != "us-east1-c" || api.deleteReq.Instance != "proxy-xyz" {
|
||||
t.Errorf("delete sent %s/%s, want us-east1-c/proxy-xyz", api.deleteReq.Zone, api.deleteReq.Instance)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListByTag(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := &fakeAPI{listInsts: []*computepb.Instance{{
|
||||
Name: proto.String("proxy-old"),
|
||||
Status: proto.String("RUNNING"),
|
||||
Zone: proto.String("https://www.googleapis.com/compute/v1/projects/my-project/zones/europe-west1-b"),
|
||||
CreationTimestamp: proto.String(time.Now().Format(time.RFC3339)),
|
||||
Labels: map[string]string{
|
||||
provider.LabelManaged: provider.LabelManagedYes,
|
||||
provider.LabelUID: "uid-orphan",
|
||||
},
|
||||
NetworkInterfaces: []*computepb.NetworkInterface{{
|
||||
AccessConfigs: []*computepb.AccessConfig{{NatIP: proto.String("34.9.9.9")}},
|
||||
}},
|
||||
}}}
|
||||
p := newTestProvider(api)
|
||||
|
||||
got, err := p.ListByTag(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ListByTag: %v", err)
|
||||
}
|
||||
|
||||
if want := "labels.proxy-operator-managed = true"; api.listReq.GetFilter() != want {
|
||||
t.Errorf("filter = %q, want %q", api.listReq.GetFilter(), want)
|
||||
}
|
||||
if !api.listReq.GetReturnPartialSuccess() {
|
||||
t.Error("ReturnPartialSuccess not set — one unreachable zone would fail the whole GC sweep")
|
||||
}
|
||||
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("instances = %d, want 1", len(got))
|
||||
}
|
||||
if got[0].ID != "zones/europe-west1-b/instances/proxy-old" {
|
||||
t.Errorf("ID = %s, want the zone parsed out of the URL-style zone field", got[0].ID)
|
||||
}
|
||||
if got[0].UID != "uid-orphan" || got[0].State != provider.StateRunning {
|
||||
t.Errorf("instance = %+v, want uid-orphan/Running", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestListByTag_errorPropagates(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := newTestProvider(&fakeAPI{listErr: gerr(500)})
|
||||
_, err := p.ListByTag(context.Background())
|
||||
if provider.Class(err) != provider.ErrTransient {
|
||||
t.Errorf("Class = %v, want ErrTransient", provider.Class(err))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseProviderID_roundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
id := formatProviderID("europe-west1-b", "proxy-abc")
|
||||
zone, name, err := parseProviderID(id)
|
||||
if err != nil || zone != "europe-west1-b" || name != "proxy-abc" {
|
||||
t.Errorf("round trip = %s/%s (%v), want europe-west1-b/proxy-abc", zone, name, err)
|
||||
}
|
||||
}
|
||||
74
internal/provider/gcp/insert.go
Normal file
74
internal/provider/gcp/insert.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package gcp
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"cloud.google.com/go/compute/apiv1/computepb"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultNetwork = "default"
|
||||
defaultNetworkTag = "proxy-operator"
|
||||
defaultDiskSizeGB = 10
|
||||
userDataKey = "user-data"
|
||||
)
|
||||
|
||||
func withDefaults(cfg provider.GCPConfig) provider.GCPConfig {
|
||||
if cfg.Network == "" {
|
||||
cfg.Network = defaultNetwork
|
||||
}
|
||||
if cfg.NetworkTag == "" {
|
||||
cfg.NetworkTag = defaultNetworkTag
|
||||
}
|
||||
if cfg.DiskSizeGB == 0 {
|
||||
cfg.DiskSizeGB = defaultDiskSizeGB
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// buildInsertRequest is pure so the field-by-field unit test needs no fake
|
||||
// at all — the plan's primary test for this provider.
|
||||
func buildInsertRequest(cfg provider.GCPConfig, req provider.CreateRequest) *computepb.InsertInstanceRequest {
|
||||
inst := &computepb.Instance{
|
||||
Name: proto.String(req.Name),
|
||||
MachineType: proto.String(fmt.Sprintf("zones/%s/machineTypes/%s", req.Placement.Zone, req.Placement.MachineType)),
|
||||
Disks: []*computepb.AttachedDisk{{
|
||||
Boot: proto.Bool(true),
|
||||
AutoDelete: proto.Bool(true),
|
||||
InitializeParams: &computepb.AttachedDiskInitializeParams{
|
||||
SourceImage: proto.String(req.Placement.Image),
|
||||
DiskSizeGb: proto.Int64(cfg.DiskSizeGB),
|
||||
},
|
||||
}},
|
||||
NetworkInterfaces: []*computepb.NetworkInterface{{
|
||||
Network: proto.String("global/networks/" + cfg.Network),
|
||||
// An ephemeral external IP: exactly this pair, per the API's
|
||||
// contract for one-to-one NAT.
|
||||
AccessConfigs: []*computepb.AccessConfig{{
|
||||
Name: proto.String("External NAT"),
|
||||
Type: proto.String("ONE_TO_ONE_NAT"),
|
||||
}},
|
||||
}},
|
||||
// The GC contract: every resource this operator creates carries
|
||||
// these two labels, and orphan GC relies on both.
|
||||
Labels: map[string]string{
|
||||
provider.LabelManaged: provider.LabelManagedYes,
|
||||
provider.LabelUID: req.UID,
|
||||
},
|
||||
Tags: &computepb.Tags{Items: []string{cfg.NetworkTag}},
|
||||
}
|
||||
if req.CloudInit != "" {
|
||||
inst.Metadata = &computepb.Metadata{Items: []*computepb.Items{{
|
||||
Key: proto.String(userDataKey),
|
||||
Value: proto.String(req.CloudInit),
|
||||
}}}
|
||||
}
|
||||
return &computepb.InsertInstanceRequest{
|
||||
Project: cfg.Project,
|
||||
Zone: req.Placement.Zone,
|
||||
InstanceResource: inst,
|
||||
}
|
||||
}
|
||||
127
internal/provider/gcp/insert_test.go
Normal file
127
internal/provider/gcp/insert_test.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package gcp
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
func testCreateRequest() provider.CreateRequest {
|
||||
return provider.CreateRequest{
|
||||
Name: "proxy-abc123def456ghij",
|
||||
UID: "11111111-2222-3333-4444-555555555555",
|
||||
Namespace: "default",
|
||||
ProxyName: "eu-proxy-1",
|
||||
Placement: provider.Placement{
|
||||
Zone: "europe-west1-b",
|
||||
MachineType: "e2-micro",
|
||||
Image: "projects/debian-cloud/global/images/family/debian-12",
|
||||
},
|
||||
CloudInit: "#cloud-config\npackages: [squid]",
|
||||
Port: 3128,
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildInsertRequest_fieldByField(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := withDefaults(provider.GCPConfig{Project: "my-project"})
|
||||
req := buildInsertRequest(cfg, testCreateRequest())
|
||||
|
||||
if req.Project != "my-project" || req.Zone != "europe-west1-b" {
|
||||
t.Errorf("project/zone = %s/%s, want my-project/europe-west1-b", req.Project, req.Zone)
|
||||
}
|
||||
inst := req.InstanceResource
|
||||
if inst.GetName() != "proxy-abc123def456ghij" {
|
||||
t.Errorf("name = %s", inst.GetName())
|
||||
}
|
||||
if got, want := inst.GetMachineType(), "zones/europe-west1-b/machineTypes/e2-micro"; got != want {
|
||||
t.Errorf("machineType = %s, want %s", got, want)
|
||||
}
|
||||
|
||||
if len(inst.GetDisks()) != 1 {
|
||||
t.Fatalf("disks = %d, want 1", len(inst.GetDisks()))
|
||||
}
|
||||
disk := inst.GetDisks()[0]
|
||||
if !disk.GetBoot() || !disk.GetAutoDelete() {
|
||||
t.Errorf("boot/autoDelete = %v/%v, want true/true", disk.GetBoot(), disk.GetAutoDelete())
|
||||
}
|
||||
if got, want := disk.GetInitializeParams().GetSourceImage(), "projects/debian-cloud/global/images/family/debian-12"; got != want {
|
||||
t.Errorf("sourceImage = %s, want %s", got, want)
|
||||
}
|
||||
if disk.GetInitializeParams().GetDiskSizeGb() != 10 {
|
||||
t.Errorf("diskSizeGb = %d, want the 10 default", disk.GetInitializeParams().GetDiskSizeGb())
|
||||
}
|
||||
|
||||
if len(inst.GetNetworkInterfaces()) != 1 {
|
||||
t.Fatalf("networkInterfaces = %d, want 1", len(inst.GetNetworkInterfaces()))
|
||||
}
|
||||
nic := inst.GetNetworkInterfaces()[0]
|
||||
if got, want := nic.GetNetwork(), "global/networks/default"; got != want {
|
||||
t.Errorf("network = %s, want %s", got, want)
|
||||
}
|
||||
if len(nic.GetAccessConfigs()) != 1 {
|
||||
t.Fatalf("accessConfigs = %d, want 1", len(nic.GetAccessConfigs()))
|
||||
}
|
||||
ac := nic.GetAccessConfigs()[0]
|
||||
if ac.GetName() != "External NAT" || ac.GetType() != "ONE_TO_ONE_NAT" {
|
||||
t.Errorf("accessConfig = %s/%s, want External NAT/ONE_TO_ONE_NAT", ac.GetName(), ac.GetType())
|
||||
}
|
||||
|
||||
wantLabels := map[string]string{
|
||||
provider.LabelManaged: provider.LabelManagedYes,
|
||||
provider.LabelUID: "11111111-2222-3333-4444-555555555555",
|
||||
}
|
||||
labels := inst.GetLabels()
|
||||
if len(labels) != len(wantLabels) {
|
||||
t.Errorf("labels = %v, want %v", labels, wantLabels)
|
||||
}
|
||||
for k, v := range wantLabels {
|
||||
if labels[k] != v {
|
||||
t.Errorf("label %s = %q, want %q", k, labels[k], v)
|
||||
}
|
||||
}
|
||||
|
||||
if tags := inst.GetTags().GetItems(); len(tags) != 1 || tags[0] != "proxy-operator" {
|
||||
t.Errorf("tags = %v, want [proxy-operator]", tags)
|
||||
}
|
||||
|
||||
items := inst.GetMetadata().GetItems()
|
||||
if len(items) != 1 || items[0].GetKey() != "user-data" {
|
||||
t.Fatalf("metadata items = %v, want one user-data entry", items)
|
||||
}
|
||||
if items[0].GetValue() != "#cloud-config\npackages: [squid]" {
|
||||
t.Errorf("user-data = %q, want the resolved cloud-init", items[0].GetValue())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildInsertRequest_configOverrides(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := withDefaults(provider.GCPConfig{
|
||||
Project: "my-project",
|
||||
Network: "crawl-vpc",
|
||||
NetworkTag: "crawl-egress",
|
||||
DiskSizeGB: 42,
|
||||
})
|
||||
req := buildInsertRequest(cfg, testCreateRequest())
|
||||
inst := req.InstanceResource
|
||||
|
||||
if got, want := inst.GetNetworkInterfaces()[0].GetNetwork(), "global/networks/crawl-vpc"; got != want {
|
||||
t.Errorf("network = %s, want %s", got, want)
|
||||
}
|
||||
if tags := inst.GetTags().GetItems(); len(tags) != 1 || tags[0] != "crawl-egress" {
|
||||
t.Errorf("tags = %v, want [crawl-egress]", tags)
|
||||
}
|
||||
if got := inst.GetDisks()[0].GetInitializeParams().GetDiskSizeGb(); got != 42 {
|
||||
t.Errorf("diskSizeGb = %d, want 42", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildInsertRequest_noCloudInitMeansNoMetadata(t *testing.T) {
|
||||
t.Parallel()
|
||||
req := testCreateRequest()
|
||||
req.CloudInit = ""
|
||||
built := buildInsertRequest(withDefaults(provider.GCPConfig{Project: "p"}), req)
|
||||
if built.InstanceResource.GetMetadata() != nil {
|
||||
t.Errorf("metadata = %v, want none without cloud-init", built.InstanceResource.GetMetadata())
|
||||
}
|
||||
}
|
||||
@@ -58,10 +58,18 @@ func buildPod(image string, req provider.CreateRequest) *corev1.Pod {
|
||||
// there, never interpreted. via/forwarded_for are turned off so the proxy
|
||||
// doesn't leak the Pod's identity to the origin.
|
||||
func squidConf(port int32) string {
|
||||
// max_filedescriptors is load-bearing in containers: squid sizes its FD
|
||||
// tables from RLIMIT_NOFILE at startup, and containerd commonly sets
|
||||
// that to effectively unlimited (kind: ~10^9) — squid then allocates
|
||||
// gigabytes and is OOM-killed before it ever listens. cache_mem is
|
||||
// trimmed because a forwarding proxy for crawling gains nothing from
|
||||
// squid's 256 MB default cache.
|
||||
return fmt.Sprintf(`http_port %d
|
||||
acl all src 0.0.0.0/0
|
||||
http_access allow all
|
||||
via off
|
||||
forwarded_for off
|
||||
max_filedescriptors 1024
|
||||
cache_mem 16 MB
|
||||
`, port)
|
||||
}
|
||||
|
||||
@@ -72,7 +72,10 @@ func TestBuildPod_usesRequestPort(t *testing.T) {
|
||||
func TestSquidConf_permissive(t *testing.T) {
|
||||
t.Parallel()
|
||||
conf := squidConf(3128)
|
||||
for _, want := range []string{"http_port 3128", "http_access allow all", "via off", "forwarded_for off"} {
|
||||
// max_filedescriptors guards against squid sizing its FD tables from a
|
||||
// container's effectively-unlimited RLIMIT_NOFILE and getting OOM-killed
|
||||
// at startup — found by the kind verification run, must not regress.
|
||||
for _, want := range []string{"http_port 3128", "http_access allow all", "via off", "forwarded_for off", "max_filedescriptors 1024"} {
|
||||
if !strings.Contains(conf, want) {
|
||||
t.Errorf("squidConf() = %q, want it to contain %q", conf, want)
|
||||
}
|
||||
|
||||
66
internal/provider/metrics.go
Normal file
66
internal/provider/metrics.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package provider
|
||||
|
||||
import "context"
|
||||
|
||||
// RequestRecorder receives one record per provider API call. Implemented
|
||||
// by internal/metrics; defined here so this package needs no metrics
|
||||
// dependency.
|
||||
type RequestRecorder interface {
|
||||
ProviderRequest(provider, op, result string)
|
||||
}
|
||||
|
||||
// WithMetrics wraps a Provider so every call is recorded with its
|
||||
// classified result — zero-cost instrumentation for the next five
|
||||
// providers, and the one place Class is called purely for observability.
|
||||
func WithMetrics(name string, p Provider, rec RequestRecorder) Provider {
|
||||
return &instrumented{name: name, inner: p, rec: rec}
|
||||
}
|
||||
|
||||
type instrumented struct {
|
||||
name string
|
||||
inner Provider
|
||||
rec RequestRecorder
|
||||
}
|
||||
|
||||
func (i *instrumented) Create(ctx context.Context, req CreateRequest) (string, error) {
|
||||
id, err := i.inner.Create(ctx, req)
|
||||
i.record("create", err)
|
||||
return id, err
|
||||
}
|
||||
|
||||
func (i *instrumented) Get(ctx context.Context, providerID string) (*Instance, error) {
|
||||
inst, err := i.inner.Get(ctx, providerID)
|
||||
i.record("get", err)
|
||||
return inst, err
|
||||
}
|
||||
|
||||
func (i *instrumented) Delete(ctx context.Context, providerID string) error {
|
||||
err := i.inner.Delete(ctx, providerID)
|
||||
i.record("delete", err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (i *instrumented) ListByTag(ctx context.Context) ([]Instance, error) {
|
||||
instances, err := i.inner.ListByTag(ctx)
|
||||
i.record("list", err)
|
||||
return instances, err
|
||||
}
|
||||
|
||||
func (i *instrumented) record(op string, err error) {
|
||||
i.rec.ProviderRequest(i.name, op, resultLabel(err))
|
||||
}
|
||||
|
||||
func resultLabel(err error) string {
|
||||
switch Class(err) {
|
||||
case nil:
|
||||
return "ok"
|
||||
case ErrNotFound:
|
||||
return "not_found"
|
||||
case ErrQuotaExceeded:
|
||||
return "quota_exceeded"
|
||||
case ErrPermanent:
|
||||
return "permanent"
|
||||
default:
|
||||
return "transient"
|
||||
}
|
||||
}
|
||||
100
internal/provider/metrics_test.go
Normal file
100
internal/provider/metrics_test.go
Normal file
@@ -0,0 +1,100 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type recordedCall struct{ provider, op, result string }
|
||||
|
||||
type fakeRecorder struct{ calls []recordedCall }
|
||||
|
||||
func (f *fakeRecorder) ProviderRequest(provider, op, result string) {
|
||||
f.calls = append(f.calls, recordedCall{provider, op, result})
|
||||
}
|
||||
|
||||
// staticProvider returns canned values; only the classification of its
|
||||
// errors matters here.
|
||||
type staticProvider struct {
|
||||
createErr, deleteErr, getErr, listErr error
|
||||
}
|
||||
|
||||
func (s *staticProvider) Create(context.Context, CreateRequest) (string, error) {
|
||||
return "id-1", s.createErr
|
||||
}
|
||||
func (s *staticProvider) Get(context.Context, string) (*Instance, error) {
|
||||
return &Instance{ID: "id-1"}, s.getErr
|
||||
}
|
||||
func (s *staticProvider) Delete(context.Context, string) error { return s.deleteErr }
|
||||
func (s *staticProvider) ListByTag(context.Context) ([]Instance, error) { return nil, s.listErr }
|
||||
|
||||
func TestWithMetrics_recordsClassifiedResults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inner *staticProvider
|
||||
call func(p Provider) error
|
||||
wantOp string
|
||||
wantResult string
|
||||
}{
|
||||
{
|
||||
name: "successful create is ok",
|
||||
inner: &staticProvider{},
|
||||
call: func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
|
||||
wantOp: "create",
|
||||
wantResult: "ok",
|
||||
},
|
||||
{
|
||||
name: "get NotFound",
|
||||
inner: &staticProvider{getErr: Wrap(ErrNotFound, "get", "x", "id-1", nil)},
|
||||
call: func(p Provider) error { _, err := p.Get(context.Background(), "id-1"); return err },
|
||||
wantOp: "get",
|
||||
wantResult: "not_found",
|
||||
},
|
||||
{
|
||||
name: "create quota",
|
||||
inner: &staticProvider{createErr: Wrap(ErrQuotaExceeded, "create", "x", "", nil)},
|
||||
call: func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
|
||||
wantOp: "create",
|
||||
wantResult: "quota_exceeded",
|
||||
},
|
||||
{
|
||||
name: "delete permanent",
|
||||
inner: &staticProvider{deleteErr: Wrap(ErrPermanent, "delete", "x", "id-1", nil)},
|
||||
call: func(p Provider) error { return p.Delete(context.Background(), "id-1") },
|
||||
wantOp: "delete",
|
||||
wantResult: "permanent",
|
||||
},
|
||||
{
|
||||
name: "unclassified list error is transient",
|
||||
inner: &staticProvider{listErr: errors.New("connection reset")},
|
||||
call: func(p Provider) error { _, err := p.ListByTag(context.Background()); return err },
|
||||
wantOp: "list",
|
||||
wantResult: "transient",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
rec := &fakeRecorder{}
|
||||
p := WithMetrics("gcp-eu", tc.inner, rec)
|
||||
|
||||
callErr := tc.call(p)
|
||||
|
||||
if len(rec.calls) != 1 {
|
||||
t.Fatalf("recorded %d calls, want 1", len(rec.calls))
|
||||
}
|
||||
want := recordedCall{provider: "gcp-eu", op: tc.wantOp, result: tc.wantResult}
|
||||
if rec.calls[0] != want {
|
||||
t.Errorf("recorded %+v, want %+v", rec.calls[0], want)
|
||||
}
|
||||
// The decorator must be transparent: errors pass through.
|
||||
if (tc.wantResult == "ok") != (callErr == nil) {
|
||||
t.Errorf("error passthrough broken: result %s but err %v", tc.wantResult, callErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user