diff --git a/docs/setup.md b/docs/setup.md index f0791538..c6ed967f 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -56,9 +56,9 @@ Whoever sets up the team deploys the services once from a checkout; everyone els privately. `Troubleshooting → Refresh status` syncs at once. From a checkout, `printf %s "$SECRET" | bun ace key set ACE_SECRET` then `bun ace directory ` does the same. -The services are cells, so they run on Cloudflare Workers or on [celld](#on-celld) on machines the -team runs. To deploy them to your Cloudflare account, give the Workers the same secret and your -tailnet's MagicDNS suffix. The services then serve only that tailnet; see +The services are cells, so they run on Cloudflare Workers, or on [celld](#on-celld) on machines the +team runs or on [Fly.io](#on-flyio). To deploy them to your Cloudflare account, give the Workers the +same secret and your tailnet's MagicDNS suffix. The services then serve only that tailnet; see [the team](architecture.md#the-team) for what that scope does and doesn't protect. ```sh @@ -110,6 +110,35 @@ Hosts then use `https://..ts.net` as the directory and `https://..ts.net:8443` as `HOSTED`. Redeploying updates running nodes without a restart. Compute still needs Cloudflare. +### On Fly.io + +[`services/fly.toml`](../services/fly.toml) runs one celld node per service on Fly.io, with a +Tigris bucket each. Tigris needs a card on the Fly account. Pick app names (`ace-directory-` +and `ace-channel-` below), then create the apps and buckets. `fly storage create` prints the +bucket's keys once; keep them to deploy from a checkout. + +```sh +for s in directory channel; do + fly apps create ace-$s- + fly storage create -a ace-$s- -n ace-$s- + fly secrets set -a ace-$s- --stage CELLD_BUCKET=s3://ace-$s- +done +``` + +Deploy each service's code to its bucket with that bucket's keys, then start its node. Set `HOSTED` +in `directory.env` to `https://ace-channel-.fly.dev`. + +```sh +export S3_ENDPOINT=https://fly.storage.tigris.dev AWS_REGION=auto +AWS_ACCESS_KEY_ID=… AWS_SECRET_ACCESS_KEY=… bun services/celld.ts channel deploy channel.env --bucket s3://ace-channel- +AWS_ACCESS_KEY_ID=… AWS_SECRET_ACCESS_KEY=… bun services/celld.ts directory deploy directory.env --bucket s3://ace-directory- +fly deploy -c services/fly.toml -a ace-channel- --ha=false +fly deploy -c services/fly.toml -a ace-directory- --ha=false +``` + +Hosts then use `https://ace-directory-.fly.dev` as the directory. The URLs are public, like +`workers.dev`, and refuse requests without the team's credential. + To try the services locally, put `ACE_SECRET`, `TAILNET`, and a model key in `services/channel/.dev.vars`, then: diff --git a/services/fly.toml b/services/fly.toml new file mode 100644 index 00000000..527cee36 --- /dev/null +++ b/services/fly.toml @@ -0,0 +1,37 @@ +# One celld node for the directory or hosted channels on Fly.io, with a Tigris bucket: +# fly deploy -c services/fly.toml -a --ha=false (see docs/setup.md). + +kill_signal = "SIGTERM" +# Above CELLD_SHUTDOWN_TOTAL_MS (40 s), so a stop can hand cells off before SIGKILL. +kill_timeout = 60 + +[build] +# The githubnext/celld fork carries the fix for denoland/celld#277. +image = "ghcr.io/githubnext/celld:0.6.2-ace.1" + +[env] +CELLD_ADDR = "0.0.0.0:8080" +# A public listener needs an explicit internal one; a single node has no peers to reach it. +CELLD_INTERNAL_ADDR = "127.0.0.1:8081" +# The bucket is the only copy of each write, as there is no peer to hold it. +CELLD_DURABILITY = "bucket" +S3_ENDPOINT = "https://fly.storage.tigris.dev" +AWS_REGION = "auto" + +[http_service] +internal_port = 8080 +force_https = true +# Alarms and hibernated sockets need a running node. +auto_stop_machines = "off" +auto_start_machines = true +min_machines_running = 1 + +[[http_service.checks]] +path = "/.well-known/celld/health" +interval = "15s" +timeout = "5s" +grace_period = "30s" + +[[vm]] +size = "shared-cpu-1x" +memory = "1gb"