mtfyun/ntfy
1
0 Fork 0
mirror of https://github.com/binwiederhier/ntfy.git synced 2026-11-10 14:44:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into pr-1969

Browse Source 
# Conflicts:
#	cmd/serve.go
#	cmd/serve_test.go
This commit is contained in:
binwiederhier committed 2026-10-30 21:27:43 +00:00
2 parents 4bee0a4fbc 1a85857a99
commit 1ca0e5a766
208 files changed
+21720 -2969
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/dependabot.yml
+6
View File
@@ -6,6 +6,12 @@ updates:
interval: "weekly"
cooldown:
default-days: 7
ignore:
# eslint 10 can't be installed until eslint-plugin-react and eslint-plugin-jsx-a11y support it
- dependency-name: "eslint"
update-types: ["version-update:semver-major"]
- dependency-name: "@eslint/js"
update-types: ["version-update:semver-major"]
- package-ecosystem: "gomod"
directory: "/"
.github/workflows/build.yaml
+13 -2
View File
@@ -5,7 +5,7 @@ on:
pull_request:
jobs:
build:
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -14,14 +14,25 @@ jobs:
with:
go-version-file: '.go-version'
- name: Install node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: '24'
cache: 'npm'
cache-dependency-path: './web/package-lock.json'
- name: Install dependencies
timeout-minutes: 10 # A stalled apt mirror must fail the run, not hang it
run: make build-deps-ubuntu
- name: Build all the things
run: make build
- name: Print build results and checksums
run: make cli-build-results
docker-build:
# Dockerfile-build is not used in releases, so build it here to catch it going stale (see #1962)
runs-on: ubuntu-24.04
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Build Dockerfile-build
run: docker build --file Dockerfile-build --tag ntfy-build .
- name: Run the image
run: docker run --rm ntfy-build --help
.github/workflows/release.yaml
+4 -3
View File
@@ -5,7 +5,7 @@ on:
- 'v[0-9]+.[0-9]+.[0-9]+'
jobs:
release:
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
services:
postgres:
image: postgres:17
@@ -31,17 +31,18 @@ jobs:
with:
go-version-file: '.go-version'
- name: Install node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: '24'
cache: 'npm'
cache-dependency-path: './web/package-lock.json'
- name: Docker login
uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
username: ${{ github.repository_owner }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- name: Install dependencies
timeout-minutes: 10 # A stalled apt mirror must fail the run, not hang it
run: make build-deps-ubuntu
- name: Build and publish
run: make release
.github/workflows/test.yaml
+3 -2
View File
@@ -5,7 +5,7 @@ on:
pull_request:
jobs:
test:
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
services:
postgres:
image: postgres:17
@@ -31,12 +31,13 @@ jobs:
with:
go-version-file: '.go-version'
- name: Install node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: '24'
cache: 'npm'
cache-dependency-path: './web/package-lock.json'
- name: Install dependencies
timeout-minutes: 10 # A stalled apt mirror must fail the run, not hang it
run: make build-deps-ubuntu
- name: Build docs (required for tests)
run: make docs
.gitignore
+2
View File
@@ -17,5 +17,7 @@ node_modules/
.DS_Store
__pycache__
web/dev-dist/
web/test-results/
web/playwright-report/
venv/
cmd/key-file.yaml
.go-version
+1 -1
View File
@@ -1 +1 @@
1.27.0
1.27.3
.goreleaser.yml
+8
View File
@@ -13,6 +13,10 @@ builds:
- "-linkmode=external -extldflags=-static -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}}"
goos: [ linux ]
goarch: [ amd64 ]
hooks:
post:
- cmd: ./scripts/check-cpu-baseline.sh "{{ .Path }}" # catch x86-64-v3 C code, see #2092
output: true
- id: ntfy_linux_armv6
binary: ntfy
env:
@@ -156,6 +160,7 @@ dockers:
goarch: amd64
build_flag_templates:
- "--platform=linux/amd64"
- "--build-arg=VERSION={{ .Version }}"
- image_templates:
- &arm64v8_image "binwiederhier/ntfy:{{ .Tag }}-arm64v8"
use: buildx
@@ -163,6 +168,7 @@ dockers:
goarch: arm64
build_flag_templates:
- "--platform=linux/arm64/v8"
- "--build-arg=VERSION={{ .Version }}"
- image_templates:
- &armv7_image "binwiederhier/ntfy:{{ .Tag }}-armv7"
use: buildx
@@ -171,6 +177,7 @@ dockers:
goarm: 7
build_flag_templates:
- "--platform=linux/arm/v7"
- "--build-arg=VERSION={{ .Version }}"
- image_templates:
- &armv6_image "binwiederhier/ntfy:{{ .Tag }}-armv6"
use: buildx
@@ -179,6 +186,7 @@ dockers:
goarm: 6
build_flag_templates:
- "--platform=linux/arm/v6"
- "--build-arg=VERSION={{ .Version }}"
docker_manifests:
- name_template: "binwiederhier/ntfy:latest"
image_templates:
Dockerfile
+3
View File
@@ -1,5 +1,7 @@
FROM alpine
ARG VERSION=dev
LABEL org.opencontainers.image.authors="philipp.heckel@gmail.com"
LABEL org.opencontainers.image.url="https://ntfy.sh/"
LABEL org.opencontainers.image.documentation="https://docs.ntfy.sh/"
@@ -8,6 +10,7 @@ LABEL org.opencontainers.image.vendor="Philipp C. Heckel"
LABEL org.opencontainers.image.licenses="Apache-2.0, GPL-2.0"
LABEL org.opencontainers.image.title="ntfy"
LABEL org.opencontainers.image.description="Send push notifications to your phone or desktop using PUT/POST"
LABEL org.opencontainers.image.version="$VERSION"
RUN apk add --no-cache tzdata
COPY ntfy /usr/bin
Dockerfile-arm
+3
View File
@@ -1,5 +1,7 @@
FROM alpine
ARG VERSION=dev
LABEL org.opencontainers.image.authors="philipp.heckel@gmail.com"
LABEL org.opencontainers.image.url="https://ntfy.sh/"
LABEL org.opencontainers.image.documentation="https://docs.ntfy.sh/"
@@ -8,6 +10,7 @@ LABEL org.opencontainers.image.vendor="Philipp C. Heckel"
LABEL org.opencontainers.image.licenses="Apache-2.0, GPL-2.0"
LABEL org.opencontainers.image.title="ntfy"
LABEL org.opencontainers.image.description="Send push notifications to your phone or desktop using PUT/POST"
LABEL org.opencontainers.image.version="$VERSION"
# Alpine does not support adding "tzdata" on ARM anymore, see
# https://github.com/binwiederhier/ntfy/issues/894
Dockerfile-build
+7 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.25-bookworm AS builder
FROM golang:1.26-bookworm AS builder
ARG VERSION=dev
ARG COMMIT=unknown
@@ -49,6 +49,12 @@ ADD ./attachment ./attachment
ADD ./mail ./mail
ADD ./s3 ./s3
ADD ./action ./action
ADD ./ban ./ban
ADD ./metrics ./metrics
ADD ./twilio ./twilio
ADD ./cluster ./cluster
ADD ./visitor ./visitor
ADD ./limit ./limit
ADD ./template/gotext ./template/gotext
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build make VERSION=$VERSION COMMIT=$COMMIT cli-linux-server
Makefile
+26 -8
View File
@@ -46,6 +46,7 @@ help:
@echo " make web-build - Actually build the web app"
@echo " make web-lint - Run eslint on the web app"
@echo " make web-test - Run vitest unit tests for the web app"
@echo " make web-e2e - Run Playwright e2e tests against a fresh local server"
@echo " make web-fmt - Run prettier on the web app"
@echo " make web-fmt-check - Run prettier on the web app, but don't change anything"
@echo
@@ -107,17 +108,28 @@ docker-dev:
# Ubuntu-specific
# The Azure apt mirrors on GitHub runners sometimes stall mid-download, and apt then waits forever.
# A timeout fails the stalled download, and the retries fetch it again.
APT_GET := sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30
#
# The Azure mirror also trickles for minutes without stalling, which no apt timeout catches, so
# GitHub runners use the main Ubuntu archive instead (a no-op elsewhere). Their ubuntu.sources points
# at a mirror list (mirror+file:/etc/apt/apt-mirrors.txt) with the Azure mirror first, so both change.
APT_SOURCES := /etc/apt/sources.list.d/ubuntu.sources /etc/apt/apt-mirrors.txt
build-deps-ubuntu:
sudo apt-get update
sudo apt-get install -y \
for f in $(APT_SOURCES); do [ ! -f $$f ] || sudo sed -i 's|azure\.archive\.ubuntu\.com|archive.ubuntu.com|g' $$f; done
$(APT_GET) update
$(APT_GET) install -y \
curl \
gcc-aarch64-linux-gnu \
gcc-arm-linux-gnueabi \
gcc-mingw-w64-x86-64 \
python3 \
python3-venv \
qemu-user \
jq
which pip3 || sudo apt-get install -y python3-pip
which pip3 || $(APT_GET) install -y python3-pip
# Documentation
@@ -171,6 +183,12 @@ web-lint:
web-test:
cd web && $(NPM) run test
# The e2e server must embed the real web app, so this builds the web app and a server binary
# (build/ntfy-e2e) first; see web/e2e/server.mjs for how it is started
web-e2e: web cli-deps-static-sites
CGO_ENABLED=1 go build -tags sqlite_omit_load_extension,osusergo,netgo -o build/ntfy-e2e .
cd web && npx playwright install chromium && $(NPM) run test:e2e
# Main server/client build
cli: cli-deps
@@ -282,22 +300,22 @@ test: cli-test web-test
testv: cli-testv web-test
cli-test: FORCE
go test $(shell go list -f '{{if .TestGoFiles}}{{.ImportPath}}{{end}}' ./... | grep -vE 'ntfy/v2/(test|examples|tools)')
go test $(shell go list -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}{{end}}' ./... | grep -vE 'ntfy/v2/(test|examples|tools)')
cli-testv: FORCE
go test -v $(shell go list -f '{{if .TestGoFiles}}{{.ImportPath}}{{end}}' ./... | grep -vE 'ntfy/v2/(test|examples|tools)')
go test -v $(shell go list -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}{{end}}' ./... | grep -vE 'ntfy/v2/(test|examples|tools)')
race: FORCE
go test -v -race $(shell go list -f '{{if .TestGoFiles}}{{.ImportPath}}{{end}}' ./... | grep -vE 'ntfy/v2/(test|examples|tools)')
go test -v -race $(shell go list -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}{{end}}' ./... | grep -vE 'ntfy/v2/(test|examples|tools)')
coverage:
mkdir -p build/coverage
go test -v -race -coverprofile=build/coverage/coverage.txt -covermode=atomic $(shell go list -f '{{if .TestGoFiles}}{{.ImportPath}}{{end}}' ./... | grep -vE 'ntfy/v2/(test|examples|tools|web)')
go test -v -race -coverprofile=build/coverage/coverage.txt -covermode=atomic $(shell go list -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}{{end}}' ./... | grep -vE 'ntfy/v2/(test|examples|tools|web)')
go tool cover -func build/coverage/coverage.txt
coverage-html:
mkdir -p build/coverage
go test -race -coverprofile=build/coverage/coverage.txt -covermode=atomic $(shell go list -f '{{if .TestGoFiles}}{{.ImportPath}}{{end}}' ./... | grep -vE 'ntfy/v2/(test|examples|tools)')
go test -race -coverprofile=build/coverage/coverage.txt -covermode=atomic $(shell go list -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}{{end}}' ./... | grep -vE 'ntfy/v2/(test|examples|tools)')
go tool cover -html build/coverage/coverage.txt
coverage-upload:
README.md
+115 -23
View File
@@ -1,16 +1,3 @@
<div align="center" markdown="1">
<sup>Special thanks to:</sup>
<br>
<br>
<a href="https://go.warp.dev/ntfy">
<img alt="Warp sponsorship" width="400" src="https://raw.githubusercontent.com/warpdotdev/brand-assets/refs/heads/main/Github/Sponsor/Warp-Github-LG-02.png">
</a>
### [Warp, built for coding with multiple AI agents.](https://go.warp.dev/ntfy)
[Available for MacOS, Linux, & Windows](https://go.warp.dev/ntfy)<br>
</div>
<hr>
![ntfy](web/public/static/images/ntfy.png)
# ntfy.sh | Send push notifications to your phone or desktop via PUT/POST
@@ -80,13 +67,7 @@ If you'd like to support the ntfy maintainers, please consider donating to [GitH
and [Liberapay](https://liberapay.com/ntfy). We would be humbled if you helped carry the server and developer
account costs. Even small donations are very much appreciated.
Thank you to our commercial sponsors, who help keep the service running and the development going:
<a href="https://m.do.co/c/442b929528db"><img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/SVG/DO_Logo_horizontal_blue.svg" width="201px"></a>
<a href="https://go.warp.dev/ntfy"><img src="https://raw.githubusercontent.com/warpdotdev/brand-assets/refs/heads/main/Logos/Warp-Wordmark-Black.png" width="160px"></a>
And a big fat **Thank You** to the individuals who have sponsored ntfy in the past, or are still sponsoring ntfy:
A big fat **Thank You** to the individuals who have sponsored ntfy in the past, or are still sponsoring ntfy:
<a href="https://github.com/neutralinsomniac"><img src="https://github.com/neutralinsomniac.png" width="40px" /></a>
<a href="https://github.com/aspyct"><img src="https://github.com/aspyct.png" width="40px" /></a>
@@ -176,7 +157,6 @@ And a big fat **Thank You** to the individuals who have sponsored ntfy in the pa
<a href="https://github.com/richardleach"><img src="https://github.com/richardleach.png" width="40px" /></a>
<a href="https://github.com/bear"><img src="https://github.com/bear.png" width="40px" /></a>
<a href="https://github.com/cminter"><img src="https://github.com/cminter.png" width="40px" /></a>
<a href="https://github.com/bahur142"><img src="https://github.com/bahur142.png" width="40px" /></a>
<a href="https://github.com/pgwiebes"><img src="https://github.com/pgwiebes.png" width="40px" /></a>
<a href="https://github.com/ralhei"><img src="https://github.com/ralhei.png" width="40px" /></a>
<a href="https://github.com/TechMDW"><img src="https://github.com/TechMDW.png" width="40px" /></a>
@@ -200,7 +180,6 @@ And a big fat **Thank You** to the individuals who have sponsored ntfy in the pa
<a href="https://github.com/Integral-Tech"><img src="https://github.com/Integral-Tech.png" width="40px" /></a>
<a href="https://github.com/TheTomik1"><img src="https://github.com/TheTomik1.png" width="40px" /></a>
<a href="https://github.com/dav23r"><img src="https://github.com/dav23r.png" width="40px" /></a>
<a href="https://github.com/stannynuytkens"><img src="https://github.com/stannynuytkens.png" width="40px" /></a>
<a href="https://github.com/danbartram"><img src="https://github.com/danbartram.png" width="40px" /></a>
<a href="https://github.com/arthurgleckler"><img src="https://github.com/arthurgleckler.png" width="40px" /></a>
<a href="https://github.com/tomroth04"><img src="https://github.com/tomroth04.png" width="40px" /></a>
@@ -223,9 +202,122 @@ And a big fat **Thank You** to the individuals who have sponsored ntfy in the pa
<a href="https://github.com/suhlig"><img src="https://github.com/suhlig.png" width="40px" /></a>
<a href="https://github.com/Proximus888"><img src="https://github.com/Proximus888.png" width="40px" /></a>
<a href="https://github.com/wielandp"><img src="https://github.com/wielandp.png" width="40px" /></a>
<a href="https://github.com/chxseh"><img src="https://github.com/chxseh.png" width="40px" /></a>
<a href="https://github.com/user8446"><img src="https://github.com/user8446.png" width="40px" /></a>
<a href="https://github.com/cdf-eagles"><img src="https://github.com/cdf-eagles.png" width="40px" /></a>
<a href="https://github.com/gabeklavans"><img src="https://github.com/gabeklavans.png" width="40px" /></a>
<a href="https://github.com/lyellick"><img src="https://github.com/lyellick.png" width="40px" /></a>
<a href="https://github.com/lukaszange"><img src="https://github.com/lukaszange.png" width="40px" /></a>
<a href="https://github.com/noon92"><img src="https://github.com/noon92.png" width="40px" /></a>
<a href="https://github.com/wljohnson05"><img src="https://github.com/wljohnson05.png" width="40px" /></a>
<a href="https://github.com/reivilibre"><img src="https://github.com/reivilibre.png" width="40px" /></a>
<a href="https://github.com/Coleslaw3557"><img src="https://github.com/Coleslaw3557.png" width="40px" /></a>
<a href="https://github.com/alexgaudon"><img src="https://github.com/alexgaudon.png" width="40px" /></a>
<a href="https://github.com/jarikmarwede"><img src="https://github.com/jarikmarwede.png" width="40px" /></a>
<a href="https://github.com/the-antz"><img src="https://github.com/the-antz.png" width="40px" /></a>
<a href="https://github.com/flag0010"><img src="https://github.com/flag0010.png" width="40px" /></a>
<a href="https://github.com/chsedotsh"><img src="https://github.com/chsedotsh.png" width="40px" /></a>
<a href="https://github.com/nanoandrew4"><img src="https://github.com/nanoandrew4.png" width="40px" /></a>
<a href="https://github.com/aashish-joshi"><img src="https://github.com/aashish-joshi.png" width="40px" /></a>
<a href="https://github.com/acantril"><img src="https://github.com/acantril.png" width="40px" /></a>
<a href="https://github.com/therobbielee"><img src="https://github.com/therobbielee.png" width="40px" /></a>
<a href="https://github.com/kdirectorate"><img src="https://github.com/kdirectorate.png" width="40px" /></a>
<a href="https://github.com/jestramiana"><img src="https://github.com/jestramiana.png" width="40px" /></a>
<a href="https://github.com/chefarbeiter"><img src="https://github.com/chefarbeiter.png" width="40px" /></a>
<a href="https://github.com/innateessence"><img src="https://github.com/innateessence.png" width="40px" /></a>
<a href="https://github.com/protok"><img src="https://github.com/protok.png" width="40px" /></a>
<a href="https://github.com/Gennadiyev"><img src="https://github.com/Gennadiyev.png" width="40px" /></a>
<a href="https://github.com/lennart-m"><img src="https://github.com/lennart-m.png" width="40px" /></a>
<a href="https://github.com/martin3000"><img src="https://github.com/martin3000.png" width="40px" /></a>
<a href="https://github.com/hockdudu"><img src="https://github.com/hockdudu.png" width="40px" /></a>
<a href="https://github.com/reedkrantz"><img src="https://github.com/reedkrantz.png" width="40px" /></a>
<a href="https://github.com/lsanchohub"><img src="https://github.com/lsanchohub.png" width="40px" /></a>
<a href="https://github.com/weikib"><img src="https://github.com/weikib.png" width="40px" /></a>
<a href="https://github.com/stoneynutcase"><img src="https://github.com/stoneynutcase.png" width="40px" /></a>
<a href="https://github.com/baudneo"><img src="https://github.com/baudneo.png" width="40px" /></a>
<a href="https://github.com/blockblue47"><img src="https://github.com/blockblue47.png" width="40px" /></a>
<a href="https://github.com/fristyxd"><img src="https://github.com/fristyxd.png" width="40px" /></a>
<a href="https://github.com/skijk"><img src="https://github.com/skijk.png" width="40px" /></a>
<a href="https://github.com/FluLights"><img src="https://github.com/FluLights.png" width="40px" /></a>
<a href="https://github.com/GioMez"><img src="https://github.com/GioMez.png" width="40px" /></a>
<a href="https://github.com/merpuya"><img src="https://github.com/merpuya.png" width="40px" /></a>
<a href="https://github.com/paralino"><img src="https://github.com/paralino.png" width="40px" /></a>
<a href="https://github.com/iamboozer"><img src="https://github.com/iamboozer.png" width="40px" /></a>
<a href="https://github.com/xaviierw"><img src="https://github.com/xaviierw.png" width="40px" /></a>
<a href="https://github.com/its-tujo"><img src="https://github.com/its-tujo.png" width="40px" /></a>
<a href="https://github.com/CR000K"><img src="https://github.com/CR000K.png" width="40px" /></a>
<a href="https://github.com/joe-mccarthy"><img src="https://github.com/joe-mccarthy.png" width="40px" /></a>
<a href="https://github.com/bub0-bub0"><img src="https://github.com/bub0-bub0.png" width="40px" /></a>
<a href="https://github.com/tmat255"><img src="https://github.com/tmat255.png" width="40px" /></a>
<a href="https://github.com/aqplemine"><img src="https://github.com/aqplemine.png" width="40px" /></a>
<a href="https://github.com/AlbedoSapiens"><img src="https://github.com/AlbedoSapiens.png" width="40px" /></a>
<a href="https://github.com/finneon-nvb"><img src="https://github.com/finneon-nvb.png" width="40px" /></a>
<a href="https://github.com/AsebWebDev"><img src="https://github.com/AsebWebDev.png" width="40px" /></a>
<a href="https://github.com/simpleetech"><img src="https://github.com/simpleetech.png" width="40px" /></a>
<a href="https://github.com/Glareascum"><img src="https://github.com/Glareascum.png" width="40px" /></a>
<a href="https://github.com/obliviance"><img src="https://github.com/obliviance.png" width="40px" /></a>
<a href="https://github.com/hippothomas"><img src="https://github.com/hippothomas.png" width="40px" /></a>
<a href="https://github.com/IamLunchbox"><img src="https://github.com/IamLunchbox.png" width="40px" /></a>
<a href="https://github.com/BenotoNet"><img src="https://github.com/BenotoNet.png" width="40px" /></a>
<a href="https://github.com/Hennbaer"><img src="https://github.com/Hennbaer.png" width="40px" /></a>
<a href="https://github.com/tomasodehnal"><img src="https://github.com/tomasodehnal.png" width="40px" /></a>
<a href="https://github.com/Kasi07"><img src="https://github.com/Kasi07.png" width="40px" /></a>
<a href="https://github.com/kanna5"><img src="https://github.com/kanna5.png" width="40px" /></a>
<a href="https://github.com/warpdotdev"><img src="https://github.com/warpdotdev.png" width="40px" /></a>
<a href="https://github.com/t3hero"><img src="https://github.com/t3hero.png" width="40px" /></a>
<a href="https://github.com/jorgenbs"><img src="https://github.com/jorgenbs.png" width="40px" /></a>
<a href="https://github.com/AdmiralNemo"><img src="https://github.com/AdmiralNemo.png" width="40px" /></a>
<a href="https://github.com/wcypierre"><img src="https://github.com/wcypierre.png" width="40px" /></a>
<a href="https://github.com/timgluz"><img src="https://github.com/timgluz.png" width="40px" /></a>
<a href="https://github.com/DavorPadovan"><img src="https://github.com/DavorPadovan.png" width="40px" /></a>
<a href="https://github.com/fapeg"><img src="https://github.com/fapeg.png" width="40px" /></a>
<a href="https://github.com/shaunhey"><img src="https://github.com/shaunhey.png" width="40px" /></a>
<a href="https://github.com/xiongzi"><img src="https://github.com/xiongzi.png" width="40px" /></a>
<a href="https://github.com/belthesar"><img src="https://github.com/belthesar.png" width="40px" /></a>
<a href="https://github.com/profbbrown"><img src="https://github.com/profbbrown.png" width="40px" /></a>
<a href="https://github.com/dodancs"><img src="https://github.com/dodancs.png" width="40px" /></a>
<a href="https://github.com/Spakman"><img src="https://github.com/Spakman.png" width="40px" /></a>
<a href="https://github.com/rubiojr"><img src="https://github.com/rubiojr.png" width="40px" /></a>
<a href="https://github.com/majoragee"><img src="https://github.com/majoragee.png" width="40px" /></a>
<a href="https://github.com/defewit"><img src="https://github.com/defewit.png" width="40px" /></a>
<a href="https://github.com/scode"><img src="https://github.com/scode.png" width="40px" /></a>
<a href="https://github.com/mvgrimes"><img src="https://github.com/mvgrimes.png" width="40px" /></a>
<a href="https://github.com/brpaz"><img src="https://github.com/brpaz.png" width="40px" /></a>
<a href="https://github.com/gregwinn"><img src="https://github.com/gregwinn.png" width="40px" /></a>
<a href="https://github.com/fazalmajid"><img src="https://github.com/fazalmajid.png" width="40px" /></a>
<a href="https://github.com/upsuper"><img src="https://github.com/upsuper.png" width="40px" /></a>
<a href="https://github.com/schneidr"><img src="https://github.com/schneidr.png" width="40px" /></a>
<a href="https://github.com/pm0u"><img src="https://github.com/pm0u.png" width="40px" /></a>
<a href="https://github.com/AlexKalopsia"><img src="https://github.com/AlexKalopsia.png" width="40px" /></a>
<a href="https://github.com/rpaterson"><img src="https://github.com/rpaterson.png" width="40px" /></a>
<a href="https://github.com/nabulator"><img src="https://github.com/nabulator.png" width="40px" /></a>
<a href="https://github.com/noenik"><img src="https://github.com/noenik.png" width="40px" /></a>
<a href="https://github.com/dasos"><img src="https://github.com/dasos.png" width="40px" /></a>
<a href="https://github.com/AntonAttano"><img src="https://github.com/AntonAttano.png" width="40px" /></a>
<a href="https://github.com/JohnFritz"><img src="https://github.com/JohnFritz.png" width="40px" /></a>
<a href="https://github.com/muety"><img src="https://github.com/muety.png" width="40px" /></a>
<a href="https://github.com/mjablon1"><img src="https://github.com/mjablon1.png" width="40px" /></a>
<a href="https://github.com/57-Wolve"><img src="https://github.com/57-Wolve.png" width="40px" /></a>
<a href="https://github.com/AceofSpades5757"><img src="https://github.com/AceofSpades5757.png" width="40px" /></a>
<a href="https://github.com/sdiggly"><img src="https://github.com/sdiggly.png" width="40px" /></a>
<a href="https://github.com/berkutta"><img src="https://github.com/berkutta.png" width="40px" /></a>
<a href="https://github.com/stevennevins"><img src="https://github.com/stevennevins.png" width="40px" /></a>
<a href="https://github.com/rohitshriwas"><img src="https://github.com/rohitshriwas.png" width="40px" /></a>
<a href="https://github.com/hostinger"><img src="https://github.com/hostinger.png" width="40px" /></a>
<a href="https://github.com/haguenau"><img src="https://github.com/haguenau.png" width="40px" /></a>
<a href="https://github.com/manuel-rhdt"><img src="https://github.com/manuel-rhdt.png" width="40px" /></a>
<a href="https://github.com/tylercasson"><img src="https://github.com/tylercasson.png" width="40px" /></a>
<a href="https://github.com/tagadvance"><img src="https://github.com/tagadvance.png" width="40px" /></a>
<a href="https://github.com/Conor0Callaghan"><img src="https://github.com/Conor0Callaghan.png" width="40px" /></a>
<a href="https://github.com/gorhack"><img src="https://github.com/gorhack.png" width="40px" /></a>
<a href="https://github.com/Raiper34"><img src="https://github.com/Raiper34.png" width="40px" /></a>
<a href="https://github.com/shaungarwood"><img src="https://github.com/shaungarwood.png" width="40px" /></a>
<a href="https://github.com/alex-ip"><img src="https://github.com/alex-ip.png" width="40px" /></a>
<a href="https://github.com/remcoboerma"><img src="https://github.com/remcoboerma.png" width="40px" /></a>
<a href="https://github.com/viliev"><img src="https://github.com/viliev.png" width="40px" /></a>
<a href="https://github.com/choozm"><img src="https://github.com/choozm.png" width="40px" /></a>
<a href="https://github.com/theProf"><img src="https://github.com/theProf.png" width="40px" /></a>
<a href="https://github.com/alidtz"><img src="https://github.com/alidtz.png" width="40px" /></a>
<a href="https://github.com/degill"><img src="https://github.com/degill.png" width="40px" /></a>
## Contributing
I welcome any contributions. Just create a PR or an issue. For larger features/ideas, please reach out
attachment/store_s3_test.go
+93 -6
View File
@@ -4,6 +4,7 @@ import (
"context"
"io"
"os"
"path"
"strings"
"sync"
"testing"
@@ -11,8 +12,96 @@ import (
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/s3"
"heckel.io/ntfy/v2/util"
)
const (
// testRoot is the shared root under which every test's key space lives, so that one listing
// can find what previous runs left behind
testRoot = "testpkg-attachment"
// staleTestObjectAge is how old a test object has to be before it is assumed to belong to a
// run that died: a test takes seconds, a CI job minutes, so nothing live is this old
staleTestObjectAge = 2 * time.Hour
)
var (
// testRunID separates concurrent runs that share the test bucket
testRunID = util.RandomString(10)
// sweepOnce runs the stale-object sweep once per test binary
sweepOnce sync.Once
)
// testKeyPrefix returns a key space belonging to this test alone. The bucket is shared with
// every other CI run, and deleteAllObjects wipes everything under the prefix it is given, so
// tests sharing one prefix delete each other's objects mid-run (seen as NoSuchKey 404s on an
// unrelated pull request).
func testKeyPrefix(t *testing.T, configuredPrefix string) string {
return path.Join(configuredPrefix, testRoot, testRunID, t.Name())
}
// sweepStaleTestObjects deletes objects left behind by runs that were killed before their
// cleanup ran. Per-run key spaces keep concurrent runs from deleting each other's objects, but
// they also mean nothing ever revisits a dead run's keys, so something has to collect them.
// Anything under the shared test root older than staleTestObjectAge cannot belong to a live run.
//
// This is a backstop for a bucket that has no expiration rule. Prefer a lifecycle rule on the
// bucket (expire the test root after a day): that also covers the case where nobody runs these
// tests for a while, and it cannot be skipped by a crash.
func sweepStaleTestObjects(t *testing.T, config *s3.Config) {
t.Helper()
rootConfig := *config // The sweep looks at every run's keys, not just this test's
rootConfig.Prefix = path.Join(config.Prefix, testRoot)
client := s3.New(&rootConfig)
objects, err := client.ListObjectsV2(context.Background())
if err != nil {
t.Logf("cannot list stale test objects, skipping sweep: %s", err) // Hygiene, not the test
return
}
stale := staleKeys(objects, time.Now())
if len(stale) == 0 {
return
}
t.Logf("deleting %d object(s) left behind by earlier runs", len(stale))
if err := client.DeleteObjects(context.Background(), stale); err != nil {
t.Logf("cannot delete stale test objects: %s", err)
}
}
// staleKeys returns the keys of objects old enough to belong to a run that died. An object whose
// modification time could not be read is left alone: a zero time would otherwise look infinitely
// old and take every live run's objects with it.
func staleKeys(objects []*s3.Object, now time.Time) []string {
stale := make([]string, 0)
for _, obj := range objects {
if !obj.LastModified.IsZero() && now.Sub(obj.LastModified) > staleTestObjectAge {
stale = append(stale, obj.Key)
}
}
return stale
}
func TestStaleKeys_OnlyCollectsWhatNoLiveRunCouldOwn(t *testing.T) {
// The sweep runs while other CI jobs are mid-test, so the age cutoff is the only thing
// keeping it from deleting a live run's objects
now := time.Now()
objects := []*s3.Object{
{Key: "live/just-written", LastModified: now},
{Key: "live/minutes-old", LastModified: now.Add(-10 * time.Minute)},
{Key: "dead/hours-old", LastModified: now.Add(-3 * time.Hour)},
{Key: "dead/days-old", LastModified: now.Add(-72 * time.Hour)},
}
require.Equal(t, []string{"dead/hours-old", "dead/days-old"}, staleKeys(objects, now))
}
func TestStaleKeys_IgnoresObjectsWithoutAModTime(t *testing.T) {
// LastModified is parsed from the listing and left zero if it could not be read; deleting on
// a zero time would delete everything
objects := []*s3.Object{{Key: "unknown-age", LastModified: time.Time{}}}
require.Empty(t, staleKeys(objects, time.Now()))
}
func TestS3Store_WriteWithPrefix(t *testing.T) {
s3URL := os.Getenv("NTFY_TEST_S3_URL")
if s3URL == "" {
@@ -20,7 +109,8 @@ func TestS3Store_WriteWithPrefix(t *testing.T) {
}
cfg, err := s3.ParseURL(s3URL)
require.Nil(t, err)
cfg.Prefix = "test-prefix"
sweepOnce.Do(func() { sweepStaleTestObjects(t, cfg) })
cfg.Prefix = testKeyPrefix(t, cfg.Prefix) // The key space under test IS a prefix
client := s3.New(cfg)
deleteAllObjects(t, client)
backend := newS3Backend(client)
@@ -53,11 +143,8 @@ func newTestRealS3Store(t *testing.T, totalSizeLimit int64) (*Store, *modTimeOve
}
cfg, err := s3.ParseURL(s3URL)
require.Nil(t, err)
if cfg.Prefix != "" {
cfg.Prefix = cfg.Prefix + "/testpkg-attachment"
} else {
cfg.Prefix = "testpkg-attachment"
}
sweepOnce.Do(func() { sweepStaleTestObjects(t, cfg) })
cfg.Prefix = testKeyPrefix(t, cfg.Prefix)
client := s3.New(cfg)
inner := newS3Backend(client)
wrapper := &modTimeOverrideBackend{backend: inner, modTimes: make(map[string]time.Time)}
client/client_test.go
+33
View File
@@ -5,8 +5,10 @@ import (
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/client"
"heckel.io/ntfy/v2/log"
"heckel.io/ntfy/v2/server"
"heckel.io/ntfy/v2/test"
"os"
"strings"
"testing"
"time"
)
@@ -101,6 +103,37 @@ func TestClient_Publish_Poll(t *testing.T) {
require.Equal(t, "some delayed message", messages[1].Message)
}
func TestClient_Publish_HeaderWithNewlines(t *testing.T) {
conf := server.NewConfig()
conf.BaseURL = "http://127.0.0.1" // Attachments require a base URL
s, port := test.StartServerWithConfig(t, conf)
defer test.StopServer(t, s, port)
c := client.New(newTestConfig(port))
// With an attachment as the body, message and title must travel as headers (see #1808)
msg, err := c.PublishReader("mytopic", strings.NewReader("file contents"),
client.WithFilename("test.txt"),
client.WithMessage("message\n with\n newlines"),
client.WithTitle("title\nwith newline"))
require.Nil(t, err)
require.Equal(t, "message\n with\n newlines", msg.Message)
require.Equal(t, "title\nwith newline", msg.Title)
require.Equal(t, "test.txt", msg.Attachment.Name)
// Unicode, and long values that are split into multiple encoded words
longMessage := strings.Repeat("Grüße aus Köln 🎉\r\n", 30)
msg, err = c.PublishReader("mytopic", strings.NewReader("file contents"),
client.WithFilename("über.txt"),
client.WithMessage(longMessage),
client.WithTitle("Grüße 🎉"),
client.WithTags([]string{"tada", "größe"}))
require.Nil(t, err)
require.Equal(t, strings.TrimSpace(longMessage), msg.Message)
require.Equal(t, "Grüße 🎉", msg.Title)
require.Equal(t, []string{"tada", "größe"}, msg.Tags)
require.Equal(t, "über.txt", msg.Attachment.Name)
}
func newTestConfig(port int) *client.Config {
c := client.NewConfig()
c.DefaultHost = fmt.Sprintf("http://127.0.0.1:%d", port)
client/options.go
+4 -2
View File
@@ -3,6 +3,7 @@ package client
import (
"fmt"
"heckel.io/ntfy/v2/util"
"mime"
"net/http"
"strings"
"time"
@@ -182,11 +183,12 @@ func WithTagsFilter(tags []string) SubscribeOption {
return WithQueryParam("tags", strings.Join(tags, ","))
}
// WithHeader is a generic option to add headers to a request
// WithHeader is a generic option to add headers to a request. Values with characters that are not
// allowed in HTTP headers (e.g. newlines) are RFC 2047 encoded, which the server decodes.
func WithHeader(header, value string) RequestOption {
return func(r *http.Request) error {
if value != "" {
r.Header.Set(header, value)
r.Header.Set(header, mime.BEncoding.Encode("utf-8", value))
}
return nil
}
client/options_test.go
+22
View File
@@ -0,0 +1,22 @@
package client_test
import (
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/client"
"net/http"
"testing"
)
func TestWithHeader_PlainValueUnchanged(t *testing.T) {
r, _ := http.NewRequest("POST", "http://127.0.0.1/mytopic", nil)
require.Nil(t, client.WithTitle("Backup done: 3 files, 0 errors (100%)\twith tab")(r))
require.Equal(t, "Backup done: 3 files, 0 errors (100%)\twith tab", r.Header.Get("X-Title"))
}
func TestWithHeader_SpecialCharsEncoded(t *testing.T) {
r, _ := http.NewRequest("POST", "http://127.0.0.1/mytopic", nil)
require.Nil(t, client.WithMessage("line 1\nline 2")(r))
require.Nil(t, client.WithTitle("Grüße 🎉")(r))
require.Equal(t, "=?utf-8?b?bGluZSAxCmxpbmUgMg==?=", r.Header.Get("X-Message"))
require.Equal(t, "=?utf-8?b?R3LDvMOfZSDwn46J?=", r.Header.Get("X-Title"))
}
cluster/client.go
+136
View File
@@ -0,0 +1,136 @@
package cluster
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
)
const (
clientTimeout = 5 * time.Second
healthMaxBytes = 4 * 1024 // Upper bound for a peer's health response
statusMaxBytes = 1024 * 1024 // Upper bound for a node's status response
)
// Client speaks the peer API to another node's cluster listener: it builds the requests, sets the
// secret and origin headers, and checks and decodes the answers. What a failure means (a lost
// batch, a retried state, an unhealthy peer) is the caller's decision.
type Client struct {
httpClient *http.Client
secret string
origin NodeID // Empty for callers that are not a node (the CLI); sent only on fan-out requests
}
// HTTPError is a peer API answer other than 200
type HTTPError struct {
URL string
StatusCode int
}
// NewClient creates a client authenticating with the shared cluster secret, as the given node
func NewClient(secret string, origin NodeID) *Client {
return &Client{
httpClient: &http.Client{Timeout: clientTimeout},
secret: secret,
origin: origin,
}
}
func (e *HTTPError) Error() string {
return fmt.Sprintf("request to %s failed with HTTP %d", e.URL, e.StatusCode)
}
// SendMessages posts a batch of NDJSON message lines to a peer's message endpoint
func (c *Client) SendMessages(ctx context.Context, baseURL string, body []byte) error {
return c.post(ctx, messageURL(baseURL), contentTypeNDJSON, body)
}
// SendState posts a JSON state envelope to a peer's state endpoint
func (c *Client) SendState(ctx context.Context, baseURL string, body []byte) error {
return c.post(ctx, stateURL(baseURL), contentTypeJSON, body)
}
// Health asks a peer whether it is healthy; any answer other than a 200 with a health body is an
// error, since a proxy, a captive portal or a parked domain answers 200 too
func (c *Client) Health(ctx context.Context, baseURL string) (bool, error) {
var health apiHealth
if err := c.get(ctx, healthURL(baseURL), healthMaxBytes, &health); err != nil {
return false, err
}
return health.Healthy, nil
}
// Status reads a node's view of the cluster from its status endpoint (baseURL, e.g. http://10.0.0.5:2587)
func (c *Client) Status(ctx context.Context, baseURL string) (*Status, error) {
var status Status
if err := c.get(ctx, statusURL(baseURL), statusMaxBytes, &status); err != nil {
return nil, err
}
return &status, nil
}
// post sends a fan-out request: authenticated with the secret, and carrying this node's ID so the
// receiver can skip its own broadcasts
func (c *Client) post(ctx context.Context, url, contentType string, body []byte) error {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", contentType)
req.Header.Set(secretHeader, c.secret)
if c.origin != "" {
req.Header.Set(originHeader, string(c.origin))
}
resp, err := c.httpClient.Do(req)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return &HTTPError{URL: url, StatusCode: resp.StatusCode}
}
return nil
}
// get reads a JSON answer from a secret-only endpoint, decoding at most limit bytes into v
func (c *Client) get(ctx context.Context, url string, limit int64, v any) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
req.Header.Set(secretHeader, c.secret)
resp, err := c.httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return &HTTPError{URL: url, StatusCode: resp.StatusCode}
}
return json.NewDecoder(io.LimitReader(resp.Body, limit)).Decode(v)
}
// messageURL derives the peer's message endpoint URL from its advertise URL.
func messageURL(advertiseURL string) string {
return strings.TrimRight(advertiseURL, "/") + MessagePath
}
// stateURL derives the peer's state endpoint URL from its advertise URL.
func stateURL(advertiseURL string) string {
return strings.TrimRight(advertiseURL, "/") + StatePath
}
// statusURL derives a node's status endpoint URL from its cluster base URL.
func statusURL(baseURL string) string {
return strings.TrimRight(baseURL, "/") + StatusPath
}
// healthURL derives the peer's health endpoint URL from its advertise URL.
func healthURL(advertiseURL string) string {
return strings.TrimRight(advertiseURL, "/") + HealthPath
}
cluster/client_test.go
+115
View File
@@ -0,0 +1,115 @@
package cluster
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// testClientRequest is what a fake peer saw of one request
type testClientRequest struct {
method, path, contentType, secret, origin, body string
}
// newTestClientServer answers every request with the given status and body, and records the request
func newTestClientServer(t *testing.T, status int, body string) (*httptest.Server, *testClientRequest) {
seen := &testClientRequest{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, _ := io.ReadAll(r.Body)
*seen = testClientRequest{r.Method, r.URL.Path, r.Header.Get("Content-Type"), r.Header.Get(secretHeader), r.Header.Get(originHeader), string(b)}
w.WriteHeader(status)
_, _ = io.WriteString(w, body)
}))
t.Cleanup(srv.Close)
return srv, seen
}
func TestClient_SendMessages(t *testing.T) {
srv, seen := newTestClientServer(t, http.StatusOK, "")
client := NewClient(testSecret, "node-a")
require.Nil(t, client.SendMessages(context.Background(), srv.URL+"/", []byte("line1\nline2\n")))
require.Equal(t, testClientRequest{http.MethodPost, MessagePath, contentTypeNDJSON, testSecret, "node-a", "line1\nline2\n"}, *seen)
}
func TestClient_SendState(t *testing.T) {
srv, seen := newTestClientServer(t, http.StatusOK, "")
client := NewClient(testSecret, "node-a")
require.Nil(t, client.SendState(context.Background(), srv.URL, []byte(`{"v":1}`)))
require.Equal(t, testClientRequest{http.MethodPost, StatePath, contentTypeJSON, testSecret, "node-a", `{"v":1}`}, *seen)
}
func TestClient_Status(t *testing.T) {
body, err := json.Marshal(&Status{NodeID: "node-b", Role: RoleActive, Leader: true, Healthy: true, Time: 1792330000, Peers: []*PeerStatus{{NodeID: "node-a", Out: PeerOutbound{BatchesFailed: 3}}}})
require.Nil(t, err)
srv, seen := newTestClientServer(t, http.StatusOK, string(body))
status, err := NewClient(testSecret, "").Status(context.Background(), srv.URL)
require.Nil(t, err)
require.NotNil(t, status)
require.Equal(t, NodeID("node-b"), status.NodeID)
require.Equal(t, int64(3), status.Peers[0].Out.BatchesFailed)
require.Equal(t, http.MethodGet, seen.method)
require.Equal(t, StatusPath, seen.path)
require.Equal(t, testSecret, seen.secret)
require.Equal(t, "", seen.origin) // Not a node: no origin header
}
func TestClient_NonOKIsHTTPError(t *testing.T) {
srv, _ := newTestClientServer(t, http.StatusUnauthorized, "")
client := NewClient("wrong", "node-a")
var httpErr *HTTPError
_, err := client.Status(context.Background(), srv.URL)
require.True(t, errors.As(err, &httpErr))
require.Equal(t, http.StatusUnauthorized, httpErr.StatusCode)
require.ErrorContains(t, err, "401")
err = client.SendMessages(context.Background(), srv.URL, []byte("x\n"))
require.True(t, errors.As(err, &httpErr))
require.Equal(t, http.StatusUnauthorized, httpErr.StatusCode)
}
func TestClient_ContextTimeout(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case <-r.Context().Done():
case <-time.After(2 * time.Second):
}
}))
t.Cleanup(srv.Close)
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
start := time.Now()
err := NewClient(testSecret, "node-a").SendMessages(ctx, srv.URL, []byte("x\n"))
require.ErrorIs(t, err, context.DeadlineExceeded)
require.Less(t, time.Since(start), time.Second)
}
func TestClient_Health(t *testing.T) {
client := NewClient(testSecret, "node-a")
srv, seen := newTestClientServer(t, http.StatusOK, `{"healthy":true}`)
healthy, err := client.Health(context.Background(), srv.URL)
require.Nil(t, err)
require.True(t, healthy)
require.Equal(t, HealthPath, seen.path)
require.Equal(t, testSecret, seen.secret)
srv, _ = newTestClientServer(t, http.StatusOK, `{"healthy":false}`)
healthy, err = client.Health(context.Background(), srv.URL)
require.Nil(t, err)
require.False(t, healthy)
srv, _ = newTestClientServer(t, http.StatusServiceUnavailable, `{"healthy":true}`)
healthy, err = client.Health(context.Background(), srv.URL)
require.Error(t, err)
require.False(t, healthy)
srv, _ = newTestClientServer(t, http.StatusOK, "OK") // A proxy or parked domain, not a ntfy node
healthy, err = client.Health(context.Background(), srv.URL)
require.Error(t, err)
require.False(t, healthy)
}
cluster/cluster.go
+174
View File
@@ -0,0 +1,174 @@
// Package cluster implements cross-node message delivery for a multi-node ntfy cluster. Nodes
// register themselves in a PostgreSQL node registry (control plane) and fan published messages
// out to each other directly over HTTP (data plane); PostgreSQL is never on the message path.
// The single-node default is the nop cluster, which does nothing.
package cluster
import (
"errors"
"fmt"
"net/http"
"time"
"heckel.io/ntfy/v2/cluster/registry"
"heckel.io/ntfy/v2/db"
"heckel.io/ntfy/v2/model"
)
// The cluster API: every kind of node-to-node communication is a path under /v1/cluster/,
// served only on the dedicated cluster listener. Future concerns (rate limit counters, stats)
// become new paths or new sections of the state envelope.
const (
// MessagePath receives batches of published messages (NDJSON, one apiMessage per line).
MessagePath = "/v1/cluster/message"
// StatePath receives peer state (JSON apiState): full subscription snapshots and
// incremental updates.
StatePath = "/v1/cluster/state"
// MembersPath lists the live cluster members (this node plus its live peers), for the
// load balancers' agents: each LB maintains its own upstream list from it, with the
// active members only.
MembersPath = "/v1/cluster/members"
// HealthPath reports a node's cluster health to its PEERS (200 healthy, 503 not): a node
// that lost its registration probes it to find out whether anywhere better exists for its
// subscribers. The server's public /v1/health is a separate endpoint for load balancers.
HealthPath = "/v1/cluster/health"
// StatusPath reports this node's view of the cluster for operators: its role and leadership,
// and per peer what it sent there and received from there (see Status)
StatusPath = "/v1/cluster/status"
)
// NodeID identifies a cluster node; it keys the registry, the per-peer queues, and the peer
// state table.
//
// Naming convention: a "node" is any cluster member in the absolute sense (identity, registry,
// config); a "peer" is another node as seen from this one (Peers, peerQueue, peerState). A peer
// IS a node, which is why peer values carry a NodeID.
type NodeID string
// Role is a node's part in the cluster; the registry owns it because leadership is derived there
type Role = registry.Role
type (
// Peer is a live peer node (see registry.Peer)
Peer = registry.Peer
// Peers is a list of live peer nodes (see registry.Peers)
Peers = registry.Peers
)
// Node roles (see registry.Role)
const (
RoleActive = registry.RoleActive
RolePassive = registry.RolePassive
)
const (
// secretHeader carries the shared secret authenticating node-to-node fan-out requests.
secretHeader = "X-Cluster-Secret"
// originHeader carries the sending node's ID on fan-out requests, so a node can skip
// requests that carry its own broadcasts (loop prevention).
originHeader = "X-Cluster-Origin"
)
// Content types of the peer API: message bodies are NDJSON (one JSON message per line, matching
// the framing of ntfy's own /topic/json subscribe stream), state bodies are plain JSON. Future
// node-to-node request types get their own paths on the cluster listener; an old node answering
// 404 on an unknown path keeps mixed-version clusters working during rolling deploys.
const (
contentTypeNDJSON = "application/x-ndjson"
contentTypeJSON = "application/json"
)
// tag is the log tag for everything cluster-related, so "tag=cluster -> trace" turns on
// per-message decisions without raising the level anywhere else
const tag = "cluster"
const (
defaultHeartbeatInterval = 3 * time.Second // How often a node refreshes its registry heartbeat
defaultNodeTTL = 30 * time.Second // A node counts as live if its heartbeat is newer than this; generous to avoid false-dead flapping (see plans)
)
// Member is one live cluster node as reported by MembersPath
type Member struct {
NodeID NodeID `json:"node_id"`
AdvertiseURL string `json:"advertise_url"`
Role Role `json:"role"` // Agents route client traffic to active nodes only
Healthy bool `json:"healthy"`
}
// Cluster fans published messages out to peer cluster nodes and receives their fan-out requests.
// Local delivery to a node's own subscribers still happens inline in the server; the cluster
// only covers the cross-node hop.
type Cluster interface {
http.Handler
// ForwardMessage sends a locally published message on to every live peer node; peers
// without a subscriber for the topic drop it. It is fire-and-forget and must not block the
// caller's request path.
ForwardMessage(m *model.Message) error
// BroadcastState pushes a state delta (first-subscriber hints, access changes, deleted users) to all
// peers. Nop single-node.
BroadcastState(state *State)
// IsLeader reports whether this node is the one that should run the cluster's singleton jobs
// (e.g. the Firebase keepaliver). It is derived from membership, not held as a lock, and it is
// a belief with a lease rather than a fence: see registry.IsLeader for what it does and does
// not promise. Always true single-node.
IsLeader() bool
// Members lists the live cluster members (this node plus its live peers); served on
// MembersPath for the load balancers' agents.
Members() []Member
// Healthy reports whether this node is fit to serve: its registry heartbeat is fresh
// enough (within NodeTTL) that peers still forward messages to it. Health checkers must
// fail open (never pull ALL nodes): during a full database outage every node reports
// unhealthy while the mesh keeps delivering on stale peer caches.
Healthy() bool
// Ready reports whether a load balancer should send clients here: an active node that is
// Healthy, or that is not but has no other active node to take its clients. The only active
// node stays ready with a stale heartbeat, since pulling it would leave clients nowhere to go.
// A draining node is never ready: it acts as passive (see Role).
Ready() bool
// Role reports the role this node acts in: its configured role (active, or passive), or passive
// while draining
Role() Role
// SetDraining marks this node as draining (moving its clients to other nodes, e.g. before a
// shutdown), or not. While draining, it acts as passive: it is not ready, does not lead, and its
// peers see it as passive. Nop single-node.
SetDraining(on bool)
// Draining reports whether this node is draining. Never single-node.
Draining() bool
// Peers returns the live peer nodes (as of the last registry refresh), excluding this node; e.g.
// Peers().HasActive() tells whether there is somewhere for this node's clients to go. None
// single-node.
Peers() Peers
// Close stops the cluster and releases its resources.
Close() error
}
// New creates the cluster for the given config: the nop cluster when clustering is disabled (the
// single-node default), or the peer-mesh cluster otherwise.
func New(conf *Config, pool *db.DB, deliver DeliverFunc) (Cluster, error) {
if !conf.Enabled {
return &nopCluster{}, nil
}
if pool == nil {
return nil, errors.New("cluster mode requires a PostgreSQL database (set database-url)")
}
if conf.AdvertiseURL == "" {
return nil, errors.New("cluster mode requires an advertise URL (set experimental-cluster-advertise-url)")
}
if conf.NodeID == "" {
return nil, errors.New("cluster mode requires a stable node ID (set experimental-cluster-node-id)")
}
if conf.Role == "" {
conf.Role = RoleActive
} else if !conf.Role.Valid() {
return nil, fmt.Errorf("invalid cluster role %q (set experimental-cluster-role to %s or %s)", conf.Role, RoleActive, RolePassive)
}
if conf.HeartbeatInterval == 0 {
conf.HeartbeatInterval = defaultHeartbeatInterval
}
if conf.NodeTTL == 0 {
conf.NodeTTL = defaultNodeTTL
}
return newMeshCluster(conf, pool, deliver)
}
cluster/cluster_mesh.go
+613
View File
@@ -0,0 +1,613 @@
package cluster
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"sort"
"sync"
"sync/atomic"
"time"
"heckel.io/ntfy/v2/cluster/registry"
"heckel.io/ntfy/v2/db"
"heckel.io/ntfy/v2/log"
"heckel.io/ntfy/v2/metrics"
"heckel.io/ntfy/v2/model"
"heckel.io/ntfy/v2/util"
)
const (
peerStatsRetention = 24 * time.Hour // How long a peer that left stays in Status
peerHealthTimeout = 2 * time.Second // Per-peer health probe while this node is unhealthy
peerQueueSize = 1024 // Bounded per-peer fan-out queue (drop on overflow)
batchMaxMessages = 100 // Flush a batch early when it reaches this many messages
batchMaxBytes = 256 * 1024 // Flush a batch early when it reaches this size
stateMaxBytes = 1024 * 1024 // Upper bound for inbound state bodies (announcements, deleted users)
)
// meshCluster fans messages out directly to peer nodes over HTTP (the data plane), using
// PostgreSQL only as a control plane: the node_registry table is both the membership mechanism
// and, read as "the live active node with the lowest id", the leader election (see
// registry.IsLeader).
// The message path never touches the database, not even for membership: see registry.Peers.
//
// A published message goes to EVERY live peer, which then drops it unless it has a subscriber
// for the topic. Routing by subscription knowledge was tried and removed: it made delivery
// correctness depend on the ordering of state messages, and at the cluster sizes this is built
// for the saving is a fraction of a few hundred KB/s (see docs/10-cluster.md).
//
// Each peer has its own bounded send queue and delivery worker, so a slow or wedged peer only
// backs up (and eventually drops) its own queue and never delays delivery to healthy peers.
type meshCluster struct {
conf *Config
deliver DeliverFunc
registry *registry.Registry
client *Client
mux *http.ServeMux // The internal peer API; Cluster is an http.Handler
queues map[NodeID]*peerQueue // per-peer send queues; reconciled against the registry
closed bool // Guards against ForwardMessage spawning new workers after Close
knownPeers map[NodeID]*registry.Peer // Peers seen in the last reconcile, for join/leave logging and fencing
stats map[NodeID]*peerStats // Per-peer batch counters for Status; kept after a peer leaves
lastRegistered time.Time // Last successful registry heartbeat, for Healthy
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
draining atomic.Bool // This node is draining and acts as passive (see SetDraining); lock-free reads, since Role is called with c.mu held
mu sync.Mutex // Protects queues, closed, knownPeers, stats and lastRegistered; serializes SetDraining
}
// newMeshCluster creates the mesh cluster: it sets up the registry schema, registers this node
// (synchronously, so it is discoverable before New returns), and starts the heartbeat loop.
// Peer delivery workers are started lazily as peers appear in the registry.
func newMeshCluster(conf *Config, pool *db.DB, deliver DeliverFunc) (*meshCluster, error) {
reg, err := registry.New(pool, string(conf.NodeID), conf.AdvertiseURL, conf.Role, conf.NodeTTL)
if err != nil {
return nil, err
}
// Register synchronously so the node is discoverable before the constructor returns; the
// heartbeat loop refreshes the registration from here on. The first peer snapshot is taken
// here too, so fan-out works from the first publish rather than the first heartbeat.
//
// This does announce the node before anything is listening for peers: the cluster listener
// binds later, in the server's Run. The window is bounded by process startup (74ms measured
// on a three-node harness during a rolling restart, one refused batch per peer), it is a
// fraction of the restart gap during which nothing is listening at all, and a refused batch
// is recovered by the delivery-gap report that follows it. Binding earlier would shrink the
// window, not close it.
if err := reg.Register(); err != nil {
return nil, err
}
peers, err := reg.Refresh()
if err != nil {
return nil, err
}
if conf.Role == RolePassive && !peers.HasActive() {
log.Tag(tag).Warn("This node is passive and no active node is live: load balancers checking /v1/ready send it no clients, and the cluster has no leader until an active node joins")
}
ctx, cancel := context.WithCancel(context.Background())
c := &meshCluster{
conf: conf,
deliver: deliver,
registry: reg,
client: NewClient(conf.Secret, conf.NodeID),
queues: make(map[NodeID]*peerQueue),
lastRegistered: time.Now(), // The synchronous Register above just succeeded
knownPeers: make(map[NodeID]*registry.Peer),
stats: make(map[NodeID]*peerStats),
ctx: ctx,
cancel: cancel,
}
c.mux = http.NewServeMux()
c.mux.HandleFunc("POST "+MessagePath, c.authenticated(c.handleMessage))
c.mux.HandleFunc("POST "+StatePath, c.authenticated(c.handleState))
c.mux.HandleFunc("GET "+MembersPath, c.secretAuthenticated(c.handleMembers))
c.mux.HandleFunc("GET "+HealthPath, c.secretAuthenticated(c.handleHealth))
c.mux.HandleFunc("GET "+StatusPath, c.secretAuthenticated(c.handleStatus))
c.wg.Add(2)
go c.heartbeatLoop()
go c.isolationLoop()
return c, nil
}
// ForwardMessage enqueues the message for delivery to every live peer node; a peer without
// subscribers for the topic drops it on arrival. Delivery is fire-and-forget via each peer's
// bounded batching queue; if a peer's queue is full the message is dropped for that peer
// (subscribers reconnect and re-poll history from the database).
func (c *meshCluster) ForwardMessage(msg *model.Message) error {
peers := c.registry.Peers() // The heartbeat's snapshot; never a database call on the publish path
if len(peers) == 0 {
return nil // Cluster of one; skip the marshal
}
data, err := marshalMessage(msg)
if err != nil {
return err
}
metrics.ClusterMessagesForwarded.Inc()
c.mu.Lock()
defer c.mu.Unlock()
if c.closed {
return nil // Shutting down; the message is dropped like any other in-flight fan-out
}
for _, p := range peers {
if !c.queueForNoLock(p).queue.TryEnqueue(data) {
metrics.ClusterQueueDropped.Inc()
c.statsForNoLock(NodeID(p.NodeID)).out.Dropped++
log.Tag(tag).Warn("Fan-out queue for peer %s full, dropping message %s", p.NodeID, msg.ID)
} else if ev := log.Tag(tag); ev.IsTrace() {
ev.Trace("Enqueued message %s (topic %s) for peer %s", msg.ID, msg.Topic, p.NodeID)
}
}
return nil
}
// BroadcastState tells all live peers about state changes that must not wait: topics that
// gained their first local subscriber (a hint for peers' cached rate-visitor lookups, never
// used for routing), changed access entries and deleted users.
func (c *meshCluster) BroadcastState(state *State) {
if len(state.AddedTopics) == 0 && len(state.DeletedUserIDs) == 0 && !state.AccessChanged {
return
}
peers := c.registry.Peers()
if len(peers) == 0 {
return
}
envelope := &apiState{
DeletedUserIDs: state.DeletedUserIDs,
AccessChanged: state.AccessChanged,
}
if len(state.AddedTopics) > 0 {
envelope.Topics = &apiStateTopics{Added: state.AddedTopics}
}
body, err := json.Marshal(envelope)
if err != nil {
return
}
log.Tag(tag).Debug("Broadcasting state (%d new topics, %d deleted users, access changed: %t) to %d peer(s)", len(state.AddedTopics), len(state.DeletedUserIDs), state.AccessChanged, len(peers))
for _, p := range peers {
go func(p *registry.Peer) {
_ = c.sendToPeer(NodeID(p.NodeID), stateURL(p.AdvertiseURL), func() error {
return c.client.SendState(c.ctx, p.AdvertiseURL, body)
})
}(p)
}
}
// Members returns this node plus the peers the registry currently considers live, for the LB
// agents. A peer's health here is this node's knowledge of it, not a probe: it is healthy if the
// registry view is still fresh, and unknown (reported unhealthy) once this node can no longer
// read the registry. An isolated node therefore reports nobody as healthy, including itself,
// which is what an agent needs to disregard its answer and keep the upstreams it has.
func (c *meshCluster) Members() []Member {
fresh := c.registry.Fresh()
members := []Member{{
NodeID: c.conf.NodeID,
AdvertiseURL: c.conf.AdvertiseURL,
Role: c.Role(),
Healthy: c.Healthy(),
}}
for _, p := range c.registry.Peers() {
members = append(members, Member{NodeID: NodeID(p.NodeID), AdvertiseURL: p.AdvertiseURL, Role: p.Role, Healthy: fresh})
}
return members
}
// Status reports this node's view of the cluster (see Status): every live peer from the registry
// snapshot, plus every peer this node exchanged batches with, each with both directions' counters.
// A link that only works one way shows as failed batches on one side and nothing received on the other.
func (c *meshCluster) Status() *Status {
now := time.Now()
status := &Status{
NodeID: c.conf.NodeID,
Role: c.Role(),
Leader: c.IsLeader(),
Healthy: c.Healthy(),
Time: now.Unix(),
}
peers := make(map[NodeID]*PeerStatus)
for _, p := range c.registry.Peers() {
peers[NodeID(p.NodeID)] = &PeerStatus{
NodeID: NodeID(p.NodeID),
AdvertiseURL: p.AdvertiseURL,
Role: p.Role,
Live: true,
HeartbeatAge: int64(now.Sub(p.HeartbeatAt).Seconds()),
}
}
c.mu.Lock()
for nodeID, s := range c.stats {
ps, ok := peers[nodeID]
if !ok {
ps = &PeerStatus{NodeID: nodeID} // Left the registry; its counters still tell what happened
peers[nodeID] = ps
}
ps.Out, ps.In, ps.LastSeen = s.out, s.in, s.lastSeen
}
for nodeID, q := range c.queues {
if ps, ok := peers[nodeID]; ok {
ps.Out.Queued = q.queue.Len()
}
}
c.mu.Unlock()
status.Peers = make([]*PeerStatus, 0, len(peers))
for _, ps := range peers {
status.Peers = append(status.Peers, ps)
}
sort.Slice(status.Peers, func(i, j int) bool {
return status.Peers[i].NodeID < status.Peers[j].NodeID
})
return status
}
// IsLeader reports whether this node currently holds singleton-job leadership.
func (c *meshCluster) IsLeader() bool {
return c.registry.IsLeader()
}
// Healthy reports whether this node's registry heartbeat is fresh enough that peers still
// forward messages to it (see the Cluster interface for the checker's fail-open duty).
func (c *meshCluster) Healthy() bool {
c.mu.Lock()
defer c.mu.Unlock()
return time.Since(c.lastRegistered) < c.conf.NodeTTL
}
// SetDraining marks this node as draining, or not: while draining, it acts as passive. It is not
// ready, gives up leadership at once, never fences, and registers as passive at its next
// heartbeat, so its peers stop counting it as a place for clients and the next leader takes over.
// Its configured role does not change.
func (c *meshCluster) SetDraining(on bool) {
c.mu.Lock() // Serializes writers, so the flag and the registered role always match
defer c.mu.Unlock()
c.draining.Store(on)
c.registry.SetRole(c.Role())
}
// Draining reports whether this node is draining
func (c *meshCluster) Draining() bool {
return c.draining.Load()
}
// Peers returns the live peer nodes as of the last registry refresh, excluding this node
func (c *meshCluster) Peers() Peers {
return c.registry.Peers()
}
// Role reports the role this node acts in: passive while draining, else its configured role
func (c *meshCluster) Role() Role {
if c.draining.Load() {
return RolePassive
}
return c.conf.Role
}
// Ready reports whether a load balancer should send clients here (see the Cluster interface). It
// reads the last peer snapshot rather than probing peers, since load balancers call it often.
func (c *meshCluster) Ready() bool {
if c.Role() != RoleActive {
return false
} else if c.Healthy() {
return true
}
c.mu.Lock()
defer c.mu.Unlock()
for _, p := range c.knownPeers {
if p.Role == RoleActive {
return false // Another active node can take this node's clients
}
}
return true
}
// Close stops the mesh: it deregisters this node, releases leadership, stops all peer workers,
// and waits for them to exit.
func (c *meshCluster) Close() error {
c.cancel() // Stops the heartbeat loop and aborts in-flight peer deliveries
// Close the peer queues so their workers flush and exit; final sends are best-effort since
// the context is already canceled (parity with fire-and-forget delivery)
c.mu.Lock()
c.closed = true
for nodeID, q := range c.queues {
q.queue.Close()
delete(c.queues, nodeID)
}
c.mu.Unlock()
// Wait for the loops BEFORE deregistering: an in-flight heartbeat's Register would otherwise
// re-insert our row right after Deregister deleted it
c.wg.Wait()
if err := c.registry.Deregister(); err != nil {
log.Tag(tag).Err(err).Warn("Failed to deregister node") // Its row goes stale on its own
}
metrics.ClusterLeader.Set(0)
return nil
}
// heartbeatLoop runs one heartbeat immediately (the ticker first fires a full interval after
// startup, and a fresh node should be leader-capable and state-visible right away), then one per
// interval until shutdown.
func (c *meshCluster) heartbeatLoop() {
defer c.wg.Done()
ticker := time.NewTicker(c.conf.HeartbeatInterval)
defer ticker.Stop()
if err := c.heartbeat(); err != nil {
log.Tag(tag).Err(err).Warn("Cluster heartbeat failed")
}
for {
select {
case <-c.ctx.Done():
return
case <-ticker.C:
if err := c.heartbeat(); err != nil {
log.Tag(tag).Err(err).Warn("Cluster heartbeat failed")
}
}
}
}
// heartbeat is one control-plane tick: refresh this node's registry row, read the live set back
// (which is also what updates this node's leadership standing), prune long-dead rows if this node
// leads, and reconcile the per-peer queues.
//
// A node that cannot even register itself aborts the tick: the remaining database work would
// fail against the same database, and everything downstream degrades safely without it:
// ForwardMessage keeps serving the last peer snapshot.
func (c *meshCluster) heartbeat() error {
if err := c.registry.Register(); err != nil {
return err
}
c.mu.Lock()
c.lastRegistered = time.Now()
c.mu.Unlock()
// Effective leadership: the lowest live active node id, with a hold-off that guarantees a
// no-leader gap on failover rather than two leaders (see registry.IsLeader)
if c.registry.IsLeader() {
metrics.ClusterLeader.Set(1)
if err := c.registry.Prune(); err != nil {
log.Tag(tag).Err(err).Warn("Failed to prune stale nodes") // Housekeeping only; not fatal for the tick
}
} else {
metrics.ClusterLeader.Set(0)
}
// A fresh read (not the cached view): heartbeat cadence bounds how long a newly joined
// peer can go unseen; ForwardMessage keeps using the cache on the hot path
peers, err := c.registry.Refresh()
if err != nil {
return err
}
c.reconcilePeers(peers)
return nil
}
// reconcilePeers aligns this node's per-peer attachments with the live peer set: it retires the
// queues (and workers) of peers that have left the registry or re-registered under a new
// advertise URL (the retired queue's remainder was headed for a dead address anyway). New and
// replacement queues are created lazily by ForwardMessage, not here, so a freshly joined peer is
// reachable immediately.
func (c *meshCluster) reconcilePeers(peers Peers) {
metrics.ClusterPeers.Set(float64(len(peers)))
alive := make(map[NodeID]*registry.Peer, len(peers))
for _, p := range peers {
alive[NodeID(p.NodeID)] = p
}
c.mu.Lock()
// Log joins and leaves (as seen through the up-to-NodeTTL-stale registry view)
for nodeID, p := range alive {
if _, ok := c.knownPeers[nodeID]; !ok {
log.Tag(tag).Info("Peer %s (%s, %s) joined the cluster", nodeID, p.AdvertiseURL, p.Role)
}
}
for nodeID := range c.knownPeers {
if _, ok := alive[nodeID]; !ok {
log.Tag(tag).Info("Peer %s left the cluster", nodeID)
}
}
c.knownPeers = alive
c.updatePeerStatsNoLock(alive)
for nodeID, q := range c.queues {
if p, ok := alive[nodeID]; !ok || q.advertiseURL != p.AdvertiseURL {
q.queue.Close() // Flushes the remainder; the worker exits when the queue is drained
delete(c.queues, nodeID)
}
}
c.mu.Unlock()
}
// isolationLoop runs the isolation check on its own ticker: a partitioned database can make
// heartbeat calls hang for a while, and Healthy is time-based, so this still notices
func (c *meshCluster) isolationLoop() {
defer c.wg.Done()
ticker := time.NewTicker(c.conf.HeartbeatInterval)
defer ticker.Stop()
for {
select {
case <-c.ctx.Done():
return
case <-ticker.C:
c.maybeIsolated()
}
}
}
// maybeIsolated calls IsolatedFunc while this node's registration is stale (peers no longer
// forward to it) but at least one known active peer is healthy.
//
// The peer probe is what makes this safe, and it answers one question: is there somewhere better
// for these clients to go? Closing them only helps if their reconnect lands on a node that still
// receives fan-out. Without the probe, a shared-database outage would make every node decide it
// is unhealthy and disconnect everyone, in a loop, every tick, while the mesh is in fact still
// delivering, because Peers serves the last snapshot and never queries. So with no healthy peer
// this does nothing: here is as good as anywhere, and the alternative is a reconnect storm.
//
// Roles narrow this further. A passive peer is not behind the load balancer, so it is nowhere
// better for clients that reconnect through it. A passive node never fences at all: its
// clients connected to it directly, and nothing would route them elsewhere. Nor does a draining
// node: it moves its clients already, gradually rather than all at once.
func (c *meshCluster) maybeIsolated() {
if c.conf.IsolatedFunc == nil || c.Role() == RolePassive || c.Healthy() {
return
}
c.mu.Lock()
urls := make([]string, 0, len(c.knownPeers))
for _, p := range c.knownPeers {
if p.Role == RoleActive {
urls = append(urls, p.AdvertiseURL)
}
}
c.mu.Unlock()
for _, url := range urls {
if c.peerHealthy(url) {
log.Tag(tag).Warn("This node lost its cluster registration while peer %s is healthy; closing local subscribers so they reconnect elsewhere", url)
c.conf.IsolatedFunc()
return
}
}
}
func (c *meshCluster) peerHealthy(url string) bool {
ctx, cancel := context.WithTimeout(c.ctx, peerHealthTimeout)
defer cancel()
healthy, err := c.client.Health(ctx, url)
return err == nil && healthy
}
// queueForNoLock returns the send queue for the given peer, creating it (and its delivery
// worker) if it does not exist yet. The caller must hold c.mu.
func (c *meshCluster) queueForNoLock(p *registry.Peer) *peerQueue {
nodeID := NodeID(p.NodeID)
q, ok := c.queues[nodeID]
if ok {
return q
}
q = &peerQueue{
advertiseURL: p.AdvertiseURL,
queue: util.NewLingerQueue(peerQueueSize, batchMaxMessages, batchMaxBytes,
func(line []byte) int { return len(line) }, c.conf.BatchLinger),
}
c.queues[nodeID] = q
c.wg.Add(1)
go c.peerWorker(nodeID, q)
return q
}
// peerWorker delivers batches of queued fan-out messages to a single peer. Batches form in the
// peer's LingerQueue (up to BatchLinger delay, flushed early on size/count caps); the worker
// exits when the queue is closed (peer left the registry, or mesh shutdown) and drained.
//
// A batch that cannot be delivered is dropped: recovering it is the delivery-gap mechanism, which
// is not in this piece yet, so a failed batch is counted and logged and nothing else.
func (c *meshCluster) peerWorker(nodeID NodeID, q *peerQueue) {
defer c.wg.Done()
for lines := range q.queue.Dequeue() {
body := assembleMessageBody(lines)
log.Tag(tag).Debug("Sending batch of %d message(s) (%d bytes) to peer %s", len(lines), len(body), nodeID)
err := c.sendToPeer(nodeID, messageURL(q.advertiseURL), func() error {
return c.client.SendMessages(c.ctx, q.advertiseURL, body)
})
if err != nil {
log.Tag(tag).Err(err).Warn("Batch to peer %s was lost", nodeID)
}
if c.ctx.Err() == nil { // A send aborted by shutdown says nothing about the link
c.recordSent(nodeID, len(lines), err)
}
metrics.ClusterBatchesSent.Inc()
}
}
// recordSent counts a message batch sent to a peer, per peer, for Status and the metrics
func (c *meshCluster) recordSent(nodeID NodeID, messages int, err error) {
now := time.Now()
c.mu.Lock()
defer c.mu.Unlock()
s := c.statsForNoLock(nodeID)
if err != nil {
s.out.BatchesFailed++
s.out.LastError = err.Error()
s.out.LastErrorAt = now.Unix()
s.out.LastFailed = true
metrics.ClusterPeerBatchesSent.WithLabelValues(string(nodeID), "error").Inc()
return
}
s.out.BatchesOK++
s.out.Messages += int64(messages)
s.out.LastOK = now.Unix()
s.out.LastFailed = false
metrics.ClusterPeerBatchesSent.WithLabelValues(string(nodeID), "ok").Inc()
metrics.ClusterPeerLastSendSuccess.WithLabelValues(string(nodeID)).Set(float64(now.Unix()))
}
// recordReceived counts a message batch received from a peer, per peer, for Status and the metrics
func (c *meshCluster) recordReceived(nodeID NodeID, messages int) {
c.mu.Lock()
defer c.mu.Unlock()
s := c.statsForNoLock(nodeID)
s.in.Batches++
s.in.Messages += int64(messages)
s.in.LastAt = time.Now().Unix()
metrics.ClusterPeerBatchesReceived.WithLabelValues(string(nodeID)).Inc()
}
// statsForNoLock returns the counters for the given peer, creating them on first use. The caller
// must hold c.mu.
func (c *meshCluster) statsForNoLock(nodeID NodeID) *peerStats {
s, ok := c.stats[nodeID]
if !ok {
s = &peerStats{lastSeen: time.Now().Unix()} // First contact counts as seen, so it can expire
c.stats[nodeID] = s
seedPeerMetrics(nodeID)
}
return s
}
// updatePeerStatsNoLock marks the live peers as seen now, and forgets peers gone for longer than
// peerStatsRetention, so a decommissioned node does not stay in Status. The caller must hold c.mu.
func (c *meshCluster) updatePeerStatsNoLock(alive map[NodeID]*registry.Peer) {
now := time.Now().Unix()
for nodeID := range alive {
c.statsForNoLock(nodeID).lastSeen = now
}
for nodeID, s := range c.stats {
if _, ok := alive[nodeID]; !ok && now-s.lastSeen > int64(peerStatsRetention.Seconds()) {
delete(c.stats, nodeID)
deletePeerMetrics(nodeID)
}
}
}
// seedPeerMetrics creates a new peer's labeled series at zero, so a peer that never succeeds still
// has a last-success sample (0, never) for an alert on its age, and the counters have a baseline
func seedPeerMetrics(nodeID NodeID) {
metrics.ClusterPeerBatchesSent.WithLabelValues(string(nodeID), "ok")
metrics.ClusterPeerBatchesSent.WithLabelValues(string(nodeID), "error")
metrics.ClusterPeerBatchesReceived.WithLabelValues(string(nodeID))
metrics.ClusterPeerLastSendSuccess.WithLabelValues(string(nodeID))
}
// deletePeerMetrics retires a forgotten peer's labeled series, so departed IDs are not exported forever
func deletePeerMetrics(nodeID NodeID) {
metrics.ClusterPeerBatchesSent.DeleteLabelValues(string(nodeID), "ok")
metrics.ClusterPeerBatchesSent.DeleteLabelValues(string(nodeID), "error")
metrics.ClusterPeerBatchesReceived.DeleteLabelValues(string(nodeID))
metrics.ClusterPeerLastSendSuccess.DeleteLabelValues(string(nodeID))
}
// sendToPeer runs one peer API POST (via the client, authenticated with the shared cluster
// secret). Failures are logged, counted and returned; the caller decides what a failure means (a
// message batch is dropped, state is retried on the next heartbeat).
func (c *meshCluster) sendToPeer(nodeID NodeID, url string, send func() error) error {
err := send()
var httpErr *HTTPError
if errors.As(err, &httpErr) {
metrics.ClusterSendErrors.Inc()
log.Tag(tag).Warn("Peer %s (%s) rejected request with HTTP %d", nodeID, url, httpErr.StatusCode)
return fmt.Errorf("peer %s rejected request with HTTP %d", nodeID, httpErr.StatusCode)
} else if err != nil {
if c.ctx.Err() == nil {
metrics.ClusterSendErrors.Inc()
log.Tag(tag).Err(err).Warn("Failed to send to peer %s (%s)", nodeID, url)
}
return err
}
return nil
}
cluster/cluster_mesh_handler.go
+129
View File
@@ -0,0 +1,129 @@
package cluster
import (
"crypto/subtle"
"encoding/json"
"io"
"net/http"
"heckel.io/ntfy/v2/log"
"heckel.io/ntfy/v2/model"
)
// ServeHTTP serves the internal peer API. Auth lives in the authenticated middleware, so every
// endpoint gets the same shared-secret and origin handling.
func (c *meshCluster) ServeHTTP(w http.ResponseWriter, r *http.Request) {
c.mux.ServeHTTP(w, r)
}
// authenticated wraps a peer API handler with the checks every endpoint needs: the shared
// secret (constant-time compare, rejected before any body is read), a present origin, and the
// origin self-skip (a request carrying this node's own traffic is acknowledged but ignored).
func (c *meshCluster) authenticated(h func(origin NodeID, w http.ResponseWriter, r *http.Request)) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if c.conf.Secret == "" || subtle.ConstantTimeCompare([]byte(r.Header.Get(secretHeader)), []byte(c.conf.Secret)) != 1 {
w.WriteHeader(http.StatusUnauthorized)
return
}
origin := NodeID(r.Header.Get(originHeader))
if origin == "" {
w.WriteHeader(http.StatusBadRequest)
return
}
if origin == c.conf.NodeID {
w.WriteHeader(http.StatusOK) // Our own traffic; nothing to do
return
}
h(origin, w, r)
}
}
// secretAuthenticated wraps a handler with the shared-secret check only. Unlike authenticated
// it expects no origin node, because the callers are the load balancers' agents, not peers.
func (c *meshCluster) secretAuthenticated(h http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if c.conf.Secret == "" || subtle.ConstantTimeCompare([]byte(r.Header.Get(secretHeader)), []byte(c.conf.Secret)) != 1 {
w.WriteHeader(http.StatusUnauthorized)
return
}
h(w, r)
}
}
// handleMessage receives a batch of peer messages (NDJSON) and streams them to local
// subscribers line by line, delivering each message as it is decoded.
func (c *meshCluster) handleMessage(origin NodeID, w http.ResponseWriter, r *http.Request) {
// A batch can exceed its byte cap by one message, plus framing overhead
maxBodyBytes := int64(batchMaxBytes) + c.conf.MaxMessageBytes + 1024
received := 0
deliver := func(m *model.Message) {
received++
if ev := log.Tag(tag); ev.IsTrace() {
ev.Trace("Delivering message %s (topic %s) from peer %s", m.ID, m.Topic, origin)
}
c.deliver(m)
}
if err := decodeMessageBody(io.LimitReader(r.Body, maxBodyBytes), int(c.conf.MaxMessageBytes), deliver); err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
log.Tag(tag).Debug("Received batch of %d message(s) from peer %s", received, origin)
c.recordReceived(origin, received)
w.WriteHeader(http.StatusOK)
}
// handleState receives a peer's state envelope and applies each section it carries.
func (c *meshCluster) handleState(origin NodeID, w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(io.LimitReader(r.Body, stateMaxBytes))
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
var state apiState
if err := json.Unmarshal(body, &state); err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
if state.Topics != nil && len(state.Topics.Added) > 0 && c.conf.TopicsAddedFunc != nil {
log.Tag(tag).Debug("Received %d announced topic(s) from peer %s", len(state.Topics.Added), origin)
c.conf.TopicsAddedFunc(state.Topics.Added)
}
if state.AccessChanged && c.conf.AccessChangedFunc != nil {
log.Tag(tag).Debug("Peer %s changed access entries, reloading the access cache", origin)
c.conf.AccessChangedFunc()
}
if len(state.DeletedUserIDs) > 0 && c.conf.UsersDeletedFunc != nil {
log.Tag(tag).Debug("Peer %s deleted %d user(s)", origin, len(state.DeletedUserIDs))
c.conf.UsersDeletedFunc(state.DeletedUserIDs)
}
w.WriteHeader(http.StatusOK)
}
// handleMembers lists the live cluster members for the load balancers' agents
func (c *meshCluster) handleMembers(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", contentTypeJSON)
if err := json.NewEncoder(w).Encode(c.Members()); err != nil {
log.Tag(tag).Err(err).Warn("Cannot write member list")
}
}
// handleStatus serves this node's view of the cluster to operators (see Status)
func (c *meshCluster) handleStatus(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", contentTypeJSON)
if err := json.NewEncoder(w).Encode(c.Status()); err != nil {
log.Tag(tag).Err(err).Warn("Cannot write cluster status")
}
}
// handleHealth answers a peer's health probe: whether this node's registration is fresh enough
// that peers still forward to it (see maybeIsolated for what a peer does with the answer).
func (c *meshCluster) handleHealth(w http.ResponseWriter, _ *http.Request) {
healthy := c.Healthy()
w.Header().Set("Content-Type", contentTypeJSON)
if !healthy {
w.WriteHeader(http.StatusServiceUnavailable)
}
if err := json.NewEncoder(w).Encode(&apiHealth{Healthy: healthy}); err != nil {
log.Tag(tag).Err(err).Warn("Cannot write health response")
}
}
cluster/cluster_mesh_handler_test.go
+294
View File
@@ -0,0 +1,294 @@
package cluster
import (
"context"
"encoding/json"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/require"
dbtest "heckel.io/ntfy/v2/db/test"
"heckel.io/ntfy/v2/metrics"
"heckel.io/ntfy/v2/model"
)
func TestMesh_PeerAPI_Auth(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
var delivered int
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, func(m *model.Message) {
delivered++
})
require.Nil(t, err)
defer mesh.Close()
frag, err := marshalMessage(model.NewDefaultMessage("mytopic", "hi"))
require.Nil(t, err)
payload := assembleMessageBody([][]byte{frag})
// Wrong secret -> 401, not delivered
rr := httptest.NewRecorder()
req := httptest.NewRequest("POST", MessagePath, strings.NewReader(string(payload)))
req.Header.Set(secretHeader, "wrong")
req.Header.Set(originHeader, "node-b")
mesh.ServeHTTP(rr, req)
require.Equal(t, 401, rr.Code)
// Missing secret -> 401, not delivered
rr = httptest.NewRecorder()
mesh.ServeHTTP(rr, httptest.NewRequest("POST", MessagePath, strings.NewReader(string(payload))))
require.Equal(t, 401, rr.Code)
require.Equal(t, 0, delivered)
// Missing origin -> 400, not delivered
rr = httptest.NewRecorder()
req = httptest.NewRequest("POST", MessagePath, strings.NewReader(string(payload)))
req.Header.Set(secretHeader, testSecret)
mesh.ServeHTTP(rr, req)
require.Equal(t, 400, rr.Code)
require.Equal(t, 0, delivered)
// Correct secret and origin -> 200, delivered
rr = httptest.NewRecorder()
req = httptest.NewRequest("POST", MessagePath, strings.NewReader(string(payload)))
req.Header.Set(secretHeader, testSecret)
req.Header.Set(originHeader, "node-b")
mesh.ServeHTTP(rr, req)
require.Equal(t, 200, rr.Code)
require.Equal(t, 1, delivered)
}
func TestMesh_PeerAPI_SelfOrigin(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
var delivered int
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, func(m *model.Message) {
delivered++
})
require.Nil(t, err)
defer mesh.Close()
// A request that carries this node's own broadcasts must not be re-delivered (loop prevention)
frag, err := marshalMessage(model.NewDefaultMessage("mytopic", "loop"))
require.Nil(t, err)
payload := assembleMessageBody([][]byte{frag})
rr := httptest.NewRecorder()
req := httptest.NewRequest("POST", MessagePath, strings.NewReader(string(payload)))
req.Header.Set(secretHeader, testSecret)
req.Header.Set(originHeader, "node-a") // Same as the receiving node's ID
mesh.ServeHTTP(rr, req)
require.Equal(t, 200, rr.Code)
require.Equal(t, 0, delivered)
}
func TestMesh_HealthEndpoint(t *testing.T) {
// Peers probe this to decide whether a node that lost its registration has somewhere better
// to send its subscribers, so it answers this node's own registration freshness, and it is
// secret-authenticated like the rest of the peer API
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil)
require.Nil(t, err)
defer mesh.Close()
rr := httptest.NewRecorder()
mesh.ServeHTTP(rr, httptest.NewRequest("GET", HealthPath, nil))
require.Equal(t, http.StatusUnauthorized, rr.Code) // No secret: rejected like every peer path
probe := func() *httptest.ResponseRecorder {
rr := httptest.NewRecorder()
req := httptest.NewRequest("GET", HealthPath, nil)
req.Header.Set(secretHeader, testSecret)
mesh.ServeHTTP(rr, req)
return rr
}
rr = probe()
require.Equal(t, http.StatusOK, rr.Code)
require.Contains(t, rr.Body.String(), `"healthy":true`)
// Registration goes stale once the database is gone, and the answer follows it
require.Nil(t, pool.Close())
waitFor(t, func() bool { return !mesh.Healthy() })
rr = probe()
require.Equal(t, http.StatusServiceUnavailable, rr.Code)
require.Contains(t, rr.Body.String(), `"healthy":false`)
}
func TestMesh_MembersEndpoint(t *testing.T) {
// The LB agents ask a node which nodes are live, so each LB can maintain its own upstream
// list instead of a central monitor pushing it. Secret-authenticated, read-only.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
registerFakePeer(t, pool, "node-peer", "http://192.168.1.50:2587")
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://192.168.1.10:2587"), pool, nil)
require.Nil(t, err)
defer mesh.Close()
waitFor(t, func() bool {
mesh.mu.Lock()
defer mesh.mu.Unlock()
return len(mesh.knownPeers) == 1
})
// Without the shared secret: rejected
rr := httptest.NewRecorder()
mesh.ServeHTTP(rr, httptest.NewRequest("GET", MembersPath, nil))
require.Equal(t, http.StatusUnauthorized, rr.Code)
// With it: this node plus its live peers, with the addresses an agent needs
rr = httptest.NewRecorder()
req := httptest.NewRequest("GET", MembersPath, nil)
req.Header.Set(secretHeader, testSecret)
mesh.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Code)
var members []Member
require.Nil(t, json.Unmarshal(rr.Body.Bytes(), &members))
byID := make(map[NodeID]Member)
for _, m := range members {
byID[m.NodeID] = m
}
require.Equal(t, 2, len(byID))
require.Equal(t, "http://192.168.1.10:2587", byID["node-a"].AdvertiseURL)
require.True(t, byID["node-a"].Healthy)
require.Equal(t, "http://192.168.1.50:2587", byID["node-peer"].AdvertiseURL)
}
func TestMesh_MembersEndpointReportsRoles(t *testing.T) {
// LB agents route client traffic to active nodes only, so every member carries its role,
// this node's from its config and each peer's from the registry
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
registerFakePeerWithRole(t, pool, "node-active", "http://192.168.1.50:2587", RoleActive)
registerFakePeerWithRole(t, pool, "node-passive", "http://192.168.1.51:2587", RolePassive)
conf := newTestMeshConfig("node-a", "http://192.168.1.10:2587")
conf.Role = RolePassive
mesh, err := newMeshCluster(conf, pool, nil)
require.Nil(t, err)
defer mesh.Close()
waitFor(t, func() bool {
mesh.mu.Lock()
defer mesh.mu.Unlock()
return len(mesh.knownPeers) == 2
})
rr := httptest.NewRecorder()
req := httptest.NewRequest("GET", MembersPath, nil)
req.Header.Set(secretHeader, testSecret)
mesh.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Code)
var members []map[string]any // Raw JSON, since the field name is the contract with the agents
require.Nil(t, json.Unmarshal(rr.Body.Bytes(), &members))
roles := make(map[string]any)
for _, m := range members {
roles[m["node_id"].(string)] = m["role"]
}
require.Equal(t, map[string]any{"node-a": "passive", "node-active": "active", "node-passive": "passive"}, roles)
}
func TestMesh_Status_RequiresSecret(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), openTestPool(t, schemaDSN), discardMessage)
require.Nil(t, err)
defer mesh.Close()
rr := httptest.NewRecorder()
mesh.ServeHTTP(rr, httptest.NewRequest("GET", StatusPath, nil))
require.Equal(t, 401, rr.Code)
rr = httptest.NewRecorder()
req := httptest.NewRequest("GET", StatusPath, nil)
req.Header.Set(secretHeader, testSecret)
mesh.ServeHTTP(rr, req)
require.Equal(t, 200, rr.Code)
var status Status
require.Nil(t, json.NewDecoder(rr.Body).Decode(&status))
require.Equal(t, NodeID("node-a"), status.NodeID)
require.Equal(t, RoleActive, status.Role)
require.True(t, status.Healthy)
require.NotZero(t, status.Time)
}
func TestMesh_Status_OneWayLinkFailure(t *testing.T) {
// Reproduces a firewall dropping one direction of a link (prod, 2026-10-18): node B cannot
// reach node A's cluster port, node A reaches node B fine. Both nodes stay healthy, so only
// the per-peer status shows it: B's batches to A all fail, and A never receives from B.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
dead, err := net.Listen("tcp", "127.0.0.1:0")
require.Nil(t, err)
deadURL := "http://" + dead.Addr().String()
dead.Close() // Nothing listens here any more: B's sends to A are refused
var receivedB atomic.Int32
var meshB *meshCluster
srvB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
meshB.ServeHTTP(w, r)
}))
defer srvB.Close()
meshA, err := newMeshCluster(newTestMeshConfig("status-a", deadURL), openTestPool(t, schemaDSN), discardMessage)
require.Nil(t, err)
defer meshA.Close()
meshB, err = newMeshCluster(newTestMeshConfig("status-b", srvB.URL), openTestPool(t, schemaDSN), func(m *model.Message) {
receivedB.Add(1)
})
require.Nil(t, err)
defer meshB.Close()
waitFor(t, func() bool {
return len(meshA.registry.Peers()) == 1 && len(meshB.registry.Peers()) == 1
})
sentErrorsBefore := testutil.ToFloat64(metrics.ClusterPeerBatchesSent.WithLabelValues("status-a", "error"))
receivedBefore := testutil.ToFloat64(metrics.ClusterPeerBatchesReceived.WithLabelValues("status-a"))
require.Nil(t, meshA.ForwardMessage(model.NewDefaultMessage("mytopic", "a to b")))
require.Nil(t, meshB.ForwardMessage(model.NewDefaultMessage("mytopic", "b to a")))
waitFor(t, func() bool {
return receivedB.Load() == 1 && peerStatus(meshB.Status(), "status-a").Out.BatchesFailed >= 1
})
// B's view: everything to A failed, A's batches arrived
statusB := meshB.Status()
require.Equal(t, NodeID("status-b"), statusB.NodeID)
a := peerStatus(statusB, "status-a")
require.True(t, a.Live)
require.Equal(t, RoleActive, a.Role)
require.Equal(t, deadURL, a.AdvertiseURL)
require.Less(t, a.HeartbeatAge, int64(10))
require.Equal(t, int64(0), a.Out.BatchesOK)
require.Equal(t, int64(0), a.Out.LastOK)
require.NotEmpty(t, a.Out.LastError)
require.NotZero(t, a.Out.LastErrorAt)
require.Equal(t, int64(1), a.In.Batches)
require.Equal(t, int64(1), a.In.Messages)
require.NotZero(t, a.In.LastAt)
// A's view: its batch to B went through, and nothing ever came back from B
b := peerStatus(meshA.Status(), "status-b")
require.Equal(t, int64(1), b.Out.BatchesOK)
require.Equal(t, int64(1), b.Out.Messages)
require.Equal(t, int64(0), b.Out.BatchesFailed)
require.NotZero(t, b.Out.LastOK)
require.Equal(t, int64(0), b.In.Batches)
require.Zero(t, b.In.LastAt)
// The same, per peer, in the metrics
require.Greater(t, testutil.ToFloat64(metrics.ClusterPeerBatchesSent.WithLabelValues("status-a", "error")), sentErrorsBefore)
require.Equal(t, receivedBefore+1, testutil.ToFloat64(metrics.ClusterPeerBatchesReceived.WithLabelValues("status-a")))
require.NotZero(t, testutil.ToFloat64(metrics.ClusterPeerLastSendSuccess.WithLabelValues("status-b")))
// And over HTTP, the way the CLI reads it
fetched, err := NewClient(testSecret, "").Status(context.Background(), srvB.URL)
require.Nil(t, err)
require.Equal(t, NodeID("status-b"), fetched.NodeID)
require.GreaterOrEqual(t, peerStatus(fetched, "status-a").Out.BatchesFailed, int64(1))
_, err = NewClient("wrong", "").Status(context.Background(), srvB.URL)
require.ErrorContains(t, err, "401")
}
func peerStatus(status *Status, nodeID NodeID) *PeerStatus {
for _, p := range status.Peers {
if p.NodeID == nodeID {
return p
}
}
return &PeerStatus{}
}
cluster/cluster_mesh_test.go
+924
View File
@@ -0,0 +1,924 @@
package cluster
import (
"bytes"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/cluster/registry"
"heckel.io/ntfy/v2/db"
"heckel.io/ntfy/v2/db/pg"
dbtest "heckel.io/ntfy/v2/db/test"
"heckel.io/ntfy/v2/log"
"heckel.io/ntfy/v2/model"
)
const (
testSecret = "s3cret"
)
// openTestPool opens a dedicated connection pool to the given test schema, so that each simulated
// node has its own pool like real nodes would.
func openTestPool(t testing.TB, dsn string) *db.DB {
host, err := pg.Open(dsn)
require.Nil(t, err)
d := db.New(host, nil)
t.Cleanup(func() { d.Close() })
return d
}
func newTestMeshConfig(nodeID, advertiseURL string) *Config {
return &Config{
Enabled: true,
NodeID: NodeID(nodeID),
AdvertiseURL: advertiseURL,
Role: RoleActive,
Secret: testSecret,
HeartbeatInterval: 100 * time.Millisecond,
// The liveness window AND the leadership hold-off. One second is the floor: heartbeats
// are stored in whole seconds, so a shorter TTL truncates the SQL cutoff to zero and a
// peer counts as live only inside the wall-clock second it registered in, which makes
// every test that waits for a peer to appear or leave a coin flip
NodeTTL: time.Second,
MaxMessageBytes: 1 << 20,
}
}
// registerFakePeer registers a fake peer via the registry (creating the table if the mesh has
// not been constructed yet): tests register fakes before the mesh boots, since its first
// heartbeat caches the peer list. The fake never refreshes its heartbeat.
func registerFakePeer(t testing.TB, pool *db.DB, nodeID NodeID, url string) {
t.Helper()
registerFakePeerWithRole(t, pool, nodeID, url, RoleActive)
}
func registerFakePeerWithRole(t testing.TB, pool *db.DB, nodeID NodeID, url string, role Role) {
t.Helper()
reg, err := registry.New(pool, string(nodeID), url, role, time.Minute)
require.Nil(t, err)
require.Nil(t, reg.Register())
}
func waitFor(t *testing.T, f func() bool) {
t.Helper()
for i := 0; i < 100; i++ {
if f() {
return
}
time.Sleep(50 * time.Millisecond)
}
t.Fatal("timed out waiting for condition")
}
func TestMesh_CrossNodeDelivery(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
poolA, poolB := openTestPool(t, schemaDSN), openTestPool(t, schemaDSN)
var mu sync.Mutex
var received []*model.Message
var meshB *meshCluster
srvB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
meshB.ServeHTTP(w, r)
}))
defer srvB.Close()
meshB, err := newMeshCluster(newTestMeshConfig("node-b", srvB.URL), poolB, func(m *model.Message) {
mu.Lock()
defer mu.Unlock()
received = append(received, m)
})
require.Nil(t, err)
defer meshB.Close()
meshA, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), poolA, func(m *model.Message) {
t.Error("node A must not receive its own relayed message")
})
require.Nil(t, err)
defer meshA.Close()
msg := model.NewDefaultMessage("mytopic", "hello cross-node")
require.Nil(t, meshA.ForwardMessage(msg))
waitFor(t, func() bool {
mu.Lock()
defer mu.Unlock()
return len(received) == 1
})
mu.Lock()
defer mu.Unlock()
require.Equal(t, "mytopic", received[0].Topic)
require.Equal(t, "hello cross-node", received[0].Message)
}
func TestMesh_SlowPeerIsolation(t *testing.T) {
// A wedged peer must not delay delivery to healthy peers: each peer has its own queue and
// delivery worker. With a shared send queue (the design this replaces), the slow peer's
// requests would occupy all delivery workers and starve the fast peer.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
var mu sync.Mutex
fastReceived := 0 // Messages, not requests: with batching, one request can carry many
srvFast := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
require.Nil(t, err)
messages, err := unmarshalMessageBody(body, 1<<20)
require.Nil(t, err)
mu.Lock()
fastReceived += len(messages)
mu.Unlock()
w.WriteHeader(http.StatusOK)
}))
defer srvFast.Close()
release := make(chan struct{})
srvSlow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
<-release // Wedged until the end of the test
w.WriteHeader(http.StatusOK)
}))
defer srvSlow.Close()
defer close(release)
// Register the fake peers before the mesh boots; its first heartbeat caches the peer list
for i, url := range []string{srvFast.URL, srvSlow.URL} {
registerFakePeer(t, pool, NodeID(fmt.Sprintf("node-fake-%d", i)), url)
}
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil)
require.Nil(t, err)
defer mesh.Close()
const n = 20
for i := 0; i < n; i++ {
require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i))))
}
waitFor(t, func() bool {
mu.Lock()
defer mu.Unlock()
return fastReceived == n
})
}
func TestMesh_BatchCoalescing(t *testing.T) {
// Messages published within the linger window arrive as batches: fewer HTTP requests than
// messages, with nothing lost. Fails against a one-request-per-message sender.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
var mu sync.Mutex
requests, messages := 0, 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
require.Nil(t, err)
decoded, err := unmarshalMessageBody(body, 1<<20)
require.Nil(t, err)
mu.Lock()
requests++
messages += len(decoded)
mu.Unlock()
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
registerFakePeer(t, pool, "node-fake", srv.URL)
conf := newTestMeshConfig("node-a", "http://127.0.0.1:1")
conf.BatchLinger = 150 * time.Millisecond
mesh, err := newMeshCluster(conf, pool, nil)
require.Nil(t, err)
defer mesh.Close()
const n = 20
for i := 0; i < n; i++ {
require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i))))
}
waitFor(t, func() bool {
mu.Lock()
defer mu.Unlock()
return messages == n
})
mu.Lock()
defer mu.Unlock()
require.Less(t, requests, 5, "expected %d messages coalesced into few requests, got %d", n, requests)
}
func TestMesh_DeadPeerRemovedAndRejoin(t *testing.T) {
// A peer that dies ungracefully (no Deregister) stops refreshing its heartbeat: after the
// TTL it no longer counts as live (no more sends), its queue/worker are reconciled away, the
// leader prunes its registry row, and a re-registered peer starts receiving again.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
var mu sync.Mutex
received := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
require.Nil(t, err)
messages, err := unmarshalMessageBody(body, 1<<20)
require.Nil(t, err)
mu.Lock()
received += len(messages)
mu.Unlock()
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
conf := newTestMeshConfig("node-a", "http://127.0.0.1:1")
conf.NodeTTL = time.Second // The floor: a shorter TTL truncates the SQL liveness cutoff to zero
mesh, err := newMeshCluster(conf, pool, nil)
require.Nil(t, err)
defer mesh.Close()
// The fake peer registers once and then "dies": its heartbeat is never refreshed
registerFakePeer(t, pool, "node-dead", srv.URL)
_, err = mesh.registry.Refresh() // See the peer now; the short TTL would otherwise expire it first
require.Nil(t, err)
require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", "while alive")))
waitFor(t, func() bool {
mu.Lock()
defer mu.Unlock()
return received == 1
})
// After the TTL, the peer is no longer live: its queue is reconciled away and its registry
// row is pruned by the leader (this mesh is the only real node, so it holds the lock)
waitFor(t, func() bool {
mesh.mu.Lock()
defer mesh.mu.Unlock()
return len(mesh.queues) == 0
})
waitFor(t, func() bool {
var count int
require.Nil(t, pool.QueryRow(`SELECT COUNT(*) FROM node_registry WHERE node_id = 'node-dead'`).Scan(&count))
return count == 0
})
require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", "while dead")))
time.Sleep(250 * time.Millisecond) // Give a wrong implementation time to deliver anyway
mu.Lock()
require.Equal(t, 1, received) // Only the first message arrived
mu.Unlock()
// The peer comes back (same node ID, fresh heartbeat) and receives messages again; the
// relay retries because the peer list is cached for up to the node TTL
registerFakePeer(t, pool, "node-dead", srv.URL)
waitFor(t, func() bool {
require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", "after rejoin")))
mu.Lock()
defer mu.Unlock()
return received > 1
})
}
func TestMesh_ForwardAfterClose(t *testing.T) {
// A ForwardMessage racing shutdown (e.g. an in-flight publish during server Stop) must not spawn
// a new peer queue and worker after Close: the worker would never exit (its queue is never
// closed) and nothing waits for it.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil)
require.Nil(t, err)
registerFakePeer(t, pool, "node-peer", "http://127.0.0.1:1")
require.Nil(t, mesh.Close())
require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", "too late"))) // Dropped silently
mesh.mu.Lock()
defer mesh.mu.Unlock()
require.Empty(t, mesh.queues)
}
func TestMesh_LeaderFailover(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
poolA, poolB := openTestPool(t, schemaDSN), openTestPool(t, schemaDSN)
meshA, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), poolA, nil)
require.Nil(t, err)
defer meshA.Close()
meshB, err := newMeshCluster(newTestMeshConfig("node-b", "http://127.0.0.1:1"), poolB, nil)
require.Nil(t, err)
defer meshB.Close()
// Exactly one node becomes leader
waitFor(t, func() bool {
return meshA.IsLeader() != meshB.IsLeader() // Exactly one
})
// The leader steps down; the follower takes over
leader, follower := meshA, meshB
if meshB.IsLeader() {
leader, follower = meshB, meshA
}
require.Nil(t, leader.Close())
waitFor(t, follower.IsLeader)
}
func TestMesh_CloseDeregisters(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil)
require.Nil(t, err)
var count int
require.Nil(t, pool.QueryRow(`SELECT COUNT(*) FROM node_registry WHERE node_id = 'node-a'`).Scan(&count))
require.Equal(t, 1, count)
require.Nil(t, mesh.Close())
require.Nil(t, pool.QueryRow(`SELECT COUNT(*) FROM node_registry WHERE node_id = 'node-a'`).Scan(&count))
require.Equal(t, 0, count)
}
func TestMesh_HealthyReflectsRegistration(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil)
require.Nil(t, err)
defer mesh.Close()
require.True(t, mesh.Healthy()) // Registered synchronously at construction
// Stale heartbeat: peers stop forwarding to this node, so it must report unhealthy
mesh.mu.Lock()
mesh.lastRegistered = time.Now().Add(-2 * mesh.conf.NodeTTL)
mesh.mu.Unlock()
require.False(t, mesh.Healthy())
// A successful heartbeat restores health
require.Nil(t, mesh.heartbeat())
require.True(t, mesh.Healthy())
}
func TestMesh_UsersDeletedBroadcast(t *testing.T) {
// Users deleted on node A must reach node B's UsersDeletedFunc; node A's own callback must not
// fire (no loop).
schemaDSN := dbtest.CreateTestPostgresSchema(t)
poolA, poolB := openTestPool(t, schemaDSN), openTestPool(t, schemaDSN)
var mu sync.Mutex
var receivedB []string
deletedA := 0
listenerB, err := net.Listen("tcp", "127.0.0.1:0")
require.Nil(t, err)
confB := newTestMeshConfig("node-b", "http://"+listenerB.Addr().String())
confB.UsersDeletedFunc = func(userIDs []string) {
mu.Lock()
defer mu.Unlock()
receivedB = append(receivedB, userIDs...)
}
meshB, err := newMeshCluster(confB, poolB, nil)
require.Nil(t, err)
defer meshB.Close()
srvB := &http.Server{Handler: meshB}
go srvB.Serve(listenerB)
defer srvB.Close()
confA := newTestMeshConfig("node-a", "http://127.0.0.1:1")
confA.UsersDeletedFunc = func(_ []string) {
mu.Lock()
defer mu.Unlock()
deletedA++
}
meshA, err := newMeshCluster(confA, poolA, nil)
require.Nil(t, err)
defer meshA.Close()
meshA.BroadcastState(&State{
DeletedUserIDs: []string{"u_deleted", "u_other"},
})
waitFor(t, func() bool {
mu.Lock()
defer mu.Unlock()
return len(receivedB) == 2
})
mu.Lock()
defer mu.Unlock()
require.Equal(t, []string{"u_deleted", "u_other"}, receivedB)
require.Equal(t, 0, deletedA)
}
func TestMesh_ForwardsEveryTopicToEveryPeer(t *testing.T) {
// Fan-out is unconditional: a peer gets every message and drops the ones it has no
// subscriber for. Nothing about a peer's subscriptions may gate delivery, which is what
// routing by subscription knowledge used to do (removed: ordering made it lose messages).
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
var mu sync.Mutex
var topics []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
require.Nil(t, err)
messages, err := unmarshalMessageBody(body, 1<<20)
require.Nil(t, err)
mu.Lock()
for _, m := range messages {
topics = append(topics, m.Topic)
}
mu.Unlock()
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
registerFakePeer(t, pool, "node-b", srv.URL)
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil)
require.Nil(t, err)
defer mesh.Close()
require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("some-topic", "one")))
require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("other-topic", "two")))
waitFor(t, func() bool {
mu.Lock()
defer mu.Unlock()
return len(topics) == 2
})
mu.Lock()
defer mu.Unlock()
require.ElementsMatch(t, []string{"some-topic", "other-topic"}, topics)
}
func TestMesh_TopicAnnouncementInvokesTopicsAddedFunc(t *testing.T) {
// A first-subscriber announcement from node A must reach node B's TopicsAddedFunc, which
// the server uses to drop a stale UnifiedPush rate-visitor miss
schemaDSN := dbtest.CreateTestPostgresSchema(t)
poolA, poolB := openTestPool(t, schemaDSN), openTestPool(t, schemaDSN)
var mu sync.Mutex
var addedB []string
listenerB, err := net.Listen("tcp", "127.0.0.1:0")
require.Nil(t, err)
confB := newTestMeshConfig("node-b", "http://"+listenerB.Addr().String())
confB.TopicsAddedFunc = func(topics []string) {
mu.Lock()
defer mu.Unlock()
addedB = append(addedB, topics...)
}
meshB, err := newMeshCluster(confB, poolB, nil)
require.Nil(t, err)
defer meshB.Close()
srvB := &http.Server{Handler: meshB}
go srvB.Serve(listenerB)
defer srvB.Close()
meshA, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), poolA, nil)
require.Nil(t, err)
defer meshA.Close()
meshA.BroadcastState(&State{AddedTopics: []string{"up123456789012"}})
waitFor(t, func() bool {
mu.Lock()
defer mu.Unlock()
return len(addedB) > 0
})
mu.Lock()
defer mu.Unlock()
require.Equal(t, []string{"up123456789012"}, addedB)
}
func TestMesh_IsolatedFuncWhenPeersHealthy(t *testing.T) {
// A node that lost its database (registration goes stale) while a peer is still healthy
// is isolated: peers stop forwarding to it, so its subscribers would silently receive
// nothing. IsolatedFunc tells the server to close them so clients reconnect elsewhere.
isolatedTest(t, http.StatusOK, `{"healthy":true}`, true)
}
func TestMesh_NoIsolatedFuncWhenNoPeerHealthy(t *testing.T) {
// Full database outage: every node is unhealthy, the mesh keeps delivering on its cached
// peer view, so subscribers must be kept (fail open)
isolatedTest(t, http.StatusServiceUnavailable, `{"healthy":false}`, false)
}
func TestMesh_NoIsolatedFuncWhenAPeerAnswers200WithoutBeingNtfy(t *testing.T) {
// A 200 is not evidence that the peer is a healthy ntfy: a misconfigured advertise URL can
// point at a proxy or an unrelated service that answers 200 for any path. Believing that
// would close this node's subscribers while nothing else can serve them, which is the one
// thing the isolation check must not do.
isolatedTest(t, http.StatusOK, "<html>hello from some proxy</html>", false)
}
func TestMesh_ActiveNodeDoesNotFenceForPassivePeer(t *testing.T) {
// A passive peer is not behind the load balancer, so it is not somewhere better for these
// clients: closing them would only send them back through the LB, a reconnect storm for
// nothing. Only a healthy active peer justifies fencing.
isolatedTestWithRoles(t, RoleActive, RolePassive, http.StatusOK, `{"healthy":true}`, false)
}
func TestMesh_PassiveNodeNeverFences(t *testing.T) {
// Nothing routes a passive node's clients anywhere else (they connected to it directly), so
// it keeps them even when it lost its registration and an active peer is healthy
isolatedTestWithRoles(t, RolePassive, RoleActive, http.StatusOK, `{"healthy":true}`, false)
}
func isolatedTest(t *testing.T, peerHealthStatus int, peerHealthBody string, wantIsolated bool) {
isolatedTestWithRoles(t, RoleActive, RoleActive, peerHealthStatus, peerHealthBody, wantIsolated)
}
func isolatedTestWithRoles(t *testing.T, role, peerRole Role, peerHealthStatus int, peerHealthBody string, wantIsolated bool) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == HealthPath {
w.WriteHeader(peerHealthStatus)
io.WriteString(w, peerHealthBody)
}
}))
defer peer.Close()
host, err := pg.Open(schemaDSN)
require.Nil(t, err)
pool := db.New(host, nil)
var isolated atomic.Int32
conf := newTestMeshConfig("node-a", "http://127.0.0.1:1")
conf.Role = role
conf.NodeTTL = time.Second // The floor: a shorter TTL truncates the SQL liveness cutoff to zero
conf.IsolatedFunc = func() { isolated.Add(1) }
// Registered last: the fake peer never heartbeats again, so its row must still be fresh
// when the mesh takes its first peer snapshot
registerFakePeerWithRole(t, openTestPool(t, schemaDSN), "node-peer", peer.URL, peerRole)
mesh, err := newMeshCluster(conf, pool, nil)
require.Nil(t, err)
defer mesh.Close()
waitFor(t, func() bool {
mesh.mu.Lock()
defer mesh.mu.Unlock()
return len(mesh.knownPeers) == 1
})
require.Equal(t, int32(0), isolated.Load()) // Healthy node: never isolated
pool.Close() // Database lost: registration fails from now on
waitFor(t, func() bool { return !mesh.Healthy() })
time.Sleep(2 * conf.HeartbeatInterval) // Give the isolation loop its chance to decide
require.Equal(t, wantIsolated, isolated.Load() > 0)
}
func TestMesh_IsolatedFuncWhenDatabaseHangs(t *testing.T) {
// A network partition makes database calls hang (dropped packets) instead of failing, which
// blocks the heartbeat loop. Isolation must still be detected (Healthy is time-based).
schemaDSN := dbtest.CreateTestPostgresSchema(t)
peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
io.WriteString(w, `{"healthy":true}`) // Answers like a healthy ntfy, which is what the probe requires
}))
defer peer.Close()
proxy := newFreezableProxy(t, schemaDSN)
var isolated atomic.Int32
conf := newTestMeshConfig("node-a", "http://127.0.0.1:1")
conf.NodeTTL = time.Second // The floor: a shorter TTL truncates the SQL liveness cutoff to zero
conf.IsolatedFunc = func() { isolated.Add(1) }
// Registered last, and through its own pool: the fake peer never heartbeats again, so its
// row must still be fresh when the mesh takes its first peer snapshot
registerFakePeer(t, openTestPool(t, schemaDSN), "node-peer", peer.URL)
mesh, err := newMeshCluster(conf, openTestPool(t, proxy.dsn), nil)
require.Nil(t, err)
defer mesh.Close()
waitFor(t, func() bool {
mesh.mu.Lock()
defer mesh.mu.Unlock()
return len(mesh.knownPeers) == 1
})
proxy.freeze()
waitFor(t, func() bool { return isolated.Load() > 0 })
}
// freezableProxy forwards TCP to the test database until frozen; then it stops moving bytes
// (and new connections hang), like a partition that drops packets
type freezableProxy struct {
dsn string
frozen atomic.Bool
}
func newFreezableProxy(t *testing.T, dsn string) *freezableProxy {
u, err := url.Parse(dsn)
require.Nil(t, err)
target := u.Host
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.Nil(t, err)
t.Cleanup(func() { listener.Close() })
p := &freezableProxy{}
u.Host = listener.Addr().String()
p.dsn = u.String()
pipe := func(dst, src net.Conn) {
buf := make([]byte, 32*1024)
for {
n, err := src.Read(buf)
for p.frozen.Load() {
time.Sleep(50 * time.Millisecond) // Hold everything, never close: a black hole
}
if err != nil {
dst.Close()
return
}
if _, err := dst.Write(buf[:n]); err != nil {
return
}
}
}
go func() {
for {
client, err := listener.Accept()
if err != nil {
return
}
server, err := net.Dial("tcp", target)
if err != nil {
client.Close()
continue
}
go pipe(server, client)
go pipe(client, server)
}
}()
return p
}
func (p *freezableProxy) freeze() {
p.frozen.Store(true)
}
func TestMesh_MembersOfAnIsolatedNodeAreNotReportedHealthy(t *testing.T) {
// A node that lost the database keeps serving its last peer view, so it answers the LB
// agents with a membership that may be minutes out of date, and used to mark every one of
// those peers healthy. An agent that happens to ask this node then pins its upstreams to a
// frozen list. The answer has to carry how much this node still knows: nothing.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
registerFakePeer(t, pool, "node-peer", "http://192.168.1.50:2587")
conf := newTestMeshConfig("node-a", "http://192.168.1.10:2587")
mesh, err := newMeshCluster(conf, pool, nil)
require.Nil(t, err)
defer mesh.Close()
waitFor(t, func() bool {
mesh.mu.Lock()
defer mesh.mu.Unlock()
return len(mesh.knownPeers) == 1
})
healthyPeers := func() int {
n := 0
for _, m := range mesh.Members() {
if m.NodeID != conf.NodeID && m.Healthy {
n++
}
}
return n
}
require.Equal(t, 1, healthyPeers()) // While the view is fresh, the peer counts
require.Nil(t, pool.Close()) // The database goes away: heartbeats and reads fail
waitFor(t, func() bool { return healthyPeers() == 0 })
for _, m := range mesh.Members() {
require.False(t, m.Healthy, "an isolated node reported %s as healthy", m.NodeID)
}
}
func TestMesh_ReadyWithoutAnActivePeer(t *testing.T) {
// The only active node (a production server with passive peers beside it) stays ready when
// its heartbeat fails: there is nowhere better for its clients, so a load balancer must not
// pull it
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
registerFakePeerWithRole(t, pool, "node-passive", "http://192.168.1.51:2587", RolePassive)
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://192.168.1.10:2587"), pool, nil)
require.Nil(t, err)
defer mesh.Close()
waitFor(t, func() bool {
mesh.mu.Lock()
defer mesh.mu.Unlock()
return len(mesh.knownPeers) == 1
})
require.Nil(t, pool.Close()) // Heartbeats fail from here on
waitFor(t, func() bool { return !mesh.Healthy() })
require.True(t, mesh.Ready())
}
func TestMesh_NotReadyWhenAnActivePeerCanTakeOver(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
registerFakePeer(t, pool, "node-active", "http://192.168.1.52:2587")
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://192.168.1.10:2587"), pool, nil)
require.Nil(t, err)
defer mesh.Close()
waitFor(t, func() bool {
mesh.mu.Lock()
defer mesh.mu.Unlock()
return len(mesh.knownPeers) == 1
})
require.True(t, mesh.Ready())
require.Nil(t, pool.Close())
waitFor(t, func() bool { return !mesh.Healthy() })
require.False(t, mesh.Ready())
}
func TestMesh_PassiveNodeWithoutAnActivePeerWarns(t *testing.T) {
// A passive node is never ready (load balancers send it nobody) and never leads, so on its own
// it is almost certainly a misconfiguration
var out bytes.Buffer
log.SetOutput(&out)
t.Cleanup(func() { log.SetOutput(os.Stderr) })
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
conf := newTestMeshConfig("node-a", "http://192.168.1.10:2587")
conf.Role = RolePassive
mesh, err := newMeshCluster(conf, pool, nil)
require.Nil(t, err)
defer mesh.Close()
require.Contains(t, out.String(), "no active node is live")
}
func TestMesh_PassiveNodeWithAnActivePeerDoesNotWarn(t *testing.T) {
var out bytes.Buffer
log.SetOutput(&out)
t.Cleanup(func() { log.SetOutput(os.Stderr) })
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
registerFakePeer(t, pool, "node-active", "http://192.168.1.52:2587")
conf := newTestMeshConfig("node-a", "http://192.168.1.10:2587")
conf.Role = RolePassive
mesh, err := newMeshCluster(conf, pool, nil)
require.Nil(t, err)
defer mesh.Close()
require.NotContains(t, out.String(), "no active node is live")
}
func TestMesh_PassiveNodeIsNeverReady(t *testing.T) {
// A passive node is healthy, but takes no clients from a load balancer, alone or not
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
conf := newTestMeshConfig("node-a", "http://192.168.1.10:2587")
conf.Role = RolePassive
mesh, err := newMeshCluster(conf, pool, nil)
require.Nil(t, err)
defer mesh.Close()
require.True(t, mesh.Healthy())
require.False(t, mesh.Ready())
}
func TestMesh_Status_PeerThatLeftShowsLastSeen(t *testing.T) {
// A peer that left the registry keeps its counters, marked not live, with the time it was last
// seen; after peerStatsRetention it is forgotten, so a decommissioned node does not linger
schemaDSN := dbtest.CreateTestPostgresSchema(t)
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), openTestPool(t, schemaDSN), discardMessage)
require.Nil(t, err)
defer mesh.Close()
before := time.Now().Unix()
mesh.reconcilePeers([]*registry.Peer{{NodeID: "node-b", AdvertiseURL: "http://127.0.0.1:1", Role: registry.RoleActive, HeartbeatAt: time.Now()}})
mesh.reconcilePeers(nil) // node-b left
b := findPeerStatus(mesh.Status(), "node-b")
require.NotNil(t, b)
require.False(t, b.Live)
require.GreaterOrEqual(t, b.LastSeen, before)
mesh.mu.Lock()
mesh.stats["node-b"].lastSeen = time.Now().Add(-peerStatsRetention - time.Minute).Unix()
mesh.mu.Unlock()
mesh.reconcilePeers(nil)
require.Nil(t, findPeerStatus(mesh.Status(), "node-b"))
}
func findPeerStatus(status *Status, nodeID NodeID) *PeerStatus {
for _, p := range status.Peers {
if p.NodeID == nodeID {
return p
}
}
return nil
}
func TestMesh_PeerMetrics_SeededForNewPeer(t *testing.T) {
// A peer whose sends all fail must still have a last-success series (0 = never), or an alert on
// its age has no sample: exactly the new node behind a one-way firewall (review of #2074)
mesh := &meshCluster{stats: make(map[NodeID]*peerStats)}
mesh.updatePeerStatsNoLock(map[NodeID]*registry.Peer{"node-never-ok": {NodeID: "node-never-ok"}})
mesh.recordSent("node-never-ok", 1, errors.New("connection refused"))
series := peerMetricSeries(t, "node-never-ok")
require.Contains(t, series, `ntfy_cluster_peer_last_send_success_timestamp_seconds{}`)
require.Contains(t, series, `ntfy_cluster_peer_batches_sent_total{result="ok"}`)
require.Contains(t, series, `ntfy_cluster_peer_batches_sent_total{result="error"}`)
require.Contains(t, series, `ntfy_cluster_peer_batches_received_total{}`)
// Seeding happens once: a later refresh does not reset a recorded success
mesh.recordSent("node-never-ok", 1, nil)
mesh.updatePeerStatsNoLock(map[NodeID]*registry.Peer{"node-never-ok": {NodeID: "node-never-ok"}})
require.NotZero(t, peerMetricValue(t, "ntfy_cluster_peer_last_send_success_timestamp_seconds", "node-never-ok"))
}
func TestMesh_PeerMetrics_RemovedWhenPeerForgotten(t *testing.T) {
// When a departed peer is forgotten, its labeled series go too, so retired IDs are not exported forever
mesh := &meshCluster{stats: make(map[NodeID]*peerStats)}
mesh.recordSent("node-retired", 1, nil)
mesh.recordReceived("node-retired", 1)
mesh.stats["node-retired"].lastSeen = time.Now().Add(-peerStatsRetention - time.Hour).Unix()
mesh.updatePeerStatsNoLock(map[NodeID]*registry.Peer{})
require.NotContains(t, mesh.stats, NodeID("node-retired"))
require.Empty(t, peerMetricSeries(t, "node-retired"))
}
// peerMetricSeries returns the metric series carrying the given peer label, as name{other labels}
func peerMetricSeries(t *testing.T, peer string) []string {
families, err := prometheus.DefaultGatherer.Gather()
require.Nil(t, err)
series := make([]string, 0)
for _, family := range families {
for _, m := range family.Metric {
matches, other := false, ""
for _, label := range m.Label {
if label.GetName() == "peer" {
matches = label.GetValue() == peer
} else {
other += fmt.Sprintf("%s=%q", label.GetName(), label.GetValue())
}
}
if matches {
series = append(series, fmt.Sprintf("%s{%s}", family.GetName(), other))
}
}
}
return series
}
func peerMetricValue(t *testing.T, name, peer string) float64 {
families, err := prometheus.DefaultGatherer.Gather()
require.Nil(t, err)
for _, family := range families {
if family.GetName() != name {
continue
}
for _, m := range family.Metric {
for _, label := range m.Label {
if label.GetName() == "peer" && label.GetValue() == peer {
return m.GetGauge().GetValue()
}
}
}
}
return 0
}
// discardMessage is the deliver callback for tests that do not look at deliveries
func discardMessage(_ *model.Message) {
// Nothing to do: these tests only check status and counters
}
func TestMesh_DrainingActsAsPassive(t *testing.T) {
// A draining node takes no clients and gives up leadership, and its peers see it as passive, so
// they no longer count it as a place for clients. Its configured role stays.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://192.168.1.10:2587"), pool, nil)
require.Nil(t, err)
defer mesh.Close()
waitFor(t, mesh.IsLeader)
require.True(t, mesh.Ready())
mesh.SetDraining(true)
require.True(t, mesh.Draining())
require.False(t, mesh.Ready())
require.False(t, mesh.IsLeader())
require.Equal(t, RolePassive, mesh.Role()) // The role it acts in; the configured one is in its config
require.Equal(t, RolePassive, mesh.Members()[0].Role)
observer, err := registry.New(pool, "observer", "http://192.168.1.99:2587", RolePassive, time.Minute)
require.Nil(t, err)
waitFor(t, func() bool {
peers, err := observer.Refresh()
require.Nil(t, err)
return len(peers) == 1 && peers[0].Role == RolePassive
})
mesh.SetDraining(false)
require.False(t, mesh.Draining())
require.True(t, mesh.Ready())
require.Equal(t, RoleActive, mesh.Role())
require.Equal(t, RoleActive, mesh.Members()[0].Role)
}
func TestMesh_PeersHasActive(t *testing.T) {
// A node only drains if another active node can take its clients
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
registerFakePeer(t, pool, "node-active", "http://192.168.1.52:2587")
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://192.168.1.10:2587"), pool, nil)
require.Nil(t, err)
defer mesh.Close()
require.True(t, mesh.Peers().HasActive())
}
func TestMesh_PassivePeerIsNoActivePeer(t *testing.T) {
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
registerFakePeerWithRole(t, pool, "node-passive", "http://192.168.1.51:2587", RolePassive)
mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://192.168.1.10:2587"), pool, nil)
require.Nil(t, err)
defer mesh.Close()
require.False(t, mesh.Peers().HasActive())
}
func TestMesh_AccessChangedBroadcast(t *testing.T) {
// Node A changed access entries: node B must be told, so it reloads its access cache; node A's
// own callback must not fire (no loop)
schemaDSN := dbtest.CreateTestPostgresSchema(t)
poolA, poolB := openTestPool(t, schemaDSN), openTestPool(t, schemaDSN)
changedA, changedB := atomic.Int32{}, atomic.Int32{}
listenerB, err := net.Listen("tcp", "127.0.0.1:0")
require.Nil(t, err)
confB := newTestMeshConfig("node-b", "http://"+listenerB.Addr().String())
confB.AccessChangedFunc = func() {
changedB.Add(1)
}
meshB, err := newMeshCluster(confB, poolB, nil)
require.Nil(t, err)
defer meshB.Close()
srvB := &http.Server{Handler: meshB}
go srvB.Serve(listenerB)
defer srvB.Close()
confA := newTestMeshConfig("node-a", "http://127.0.0.1:1")
confA.AccessChangedFunc = func() {
changedA.Add(1)
}
meshA, err := newMeshCluster(confA, poolA, nil)
require.Nil(t, err)
defer meshA.Close()
meshA.BroadcastState(&State{
AccessChanged: true,
})
waitFor(t, func() bool {
return changedB.Load() == 1
})
require.Equal(t, int32(0), changedA.Load())
}
cluster/cluster_nop.go
+60
View File
@@ -0,0 +1,60 @@
package cluster
import (
"net/http"
"heckel.io/ntfy/v2/model"
)
// nopCluster is the single-node default: it drops all relayed messages, rejects peer API requests, and
// reports this node as leader (a single node is trivially the leader, so leader-gated jobs need
// no special-casing in single-node mode).
type nopCluster struct{}
func (c *nopCluster) ForwardMessage(_ *model.Message) error {
return nil // No peers to forward to
}
func (c *nopCluster) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNotFound) // The peer API does not exist on a single node
}
func (c *nopCluster) BroadcastState(_ *State) {
// No peers to tell
}
func (c *nopCluster) IsLeader() bool {
return true // A single node is trivially the leader
}
func (c *nopCluster) Members() []Member {
return nil // No registry, so no membership to report
}
func (c *nopCluster) Healthy() bool {
return true // Health here means "the registry heartbeat is fresh", and there is no registry
}
func (c *nopCluster) Role() Role {
return RoleActive // A single node takes all client traffic
}
func (c *nopCluster) Draining() bool {
return false // See SetDraining
}
func (c *nopCluster) Peers() Peers {
return nil // A single node has no peers
}
func (c *nopCluster) SetDraining(_ bool) {
// Nothing to tell: there are no peers, and a single node never drains
}
func (c *nopCluster) Ready() bool {
return true // A single node takes all client traffic
}
func (c *nopCluster) Close() error {
return nil // Nothing was started
}
cluster/cluster_nop_test.go
+18
View File
@@ -0,0 +1,18 @@
package cluster
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestNop_NeverDrainsAndHasNoPeers(t *testing.T) {
// A single node has nowhere to send its clients, so it never drains
c := &nopCluster{}
c.SetDraining(true)
require.False(t, c.Draining())
require.Empty(t, c.Peers())
require.False(t, c.Peers().HasActive())
require.Equal(t, RoleActive, c.Role())
require.True(t, c.Ready())
}
cluster/cluster_test.go
+265
View File
@@ -0,0 +1,265 @@
package cluster
import (
"bytes"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/netip"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
dbtest "heckel.io/ntfy/v2/db/test"
"heckel.io/ntfy/v2/model"
)
func TestDeliver_RoundTrip(t *testing.T) {
// The fan-out body is NDJSON: one apiDeliverMessage per line, joined from pre-marshaled
// fragments; the origin travels in a header, not the body
m1 := model.NewDefaultMessage("mytopic", "my message")
m1.Sender = netip.MustParseAddr("1.2.3.4")
m1.User = "u_abc"
m2 := model.NewDefaultMessage("othertopic", "other message")
frag1, err := marshalMessage(m1)
require.Nil(t, err)
frag2, err := marshalMessage(m2)
require.Nil(t, err)
messages, err := unmarshalMessageBody(assembleMessageBody([][]byte{frag1, frag2}), 1<<20)
require.Nil(t, err)
require.Len(t, messages, 2)
require.Equal(t, "mytopic", messages[0].Topic)
require.Equal(t, "my message", messages[0].Message)
// Sender and User are json:"-" on model.Message; the lines must carry and reattach them
require.Equal(t, netip.MustParseAddr("1.2.3.4"), messages[0].Sender)
require.Equal(t, "u_abc", messages[0].User)
require.Equal(t, "othertopic", messages[1].Topic)
require.False(t, messages[1].Sender.IsValid())
}
func TestDeliver_SingleMessage(t *testing.T) {
// A single message is just a one-line body; there is no separate single-message format
frag, err := marshalMessage(model.NewDefaultMessage("mytopic", "hi"))
require.Nil(t, err)
messages, err := unmarshalMessageBody(assembleMessageBody([][]byte{frag}), 1<<20)
require.Nil(t, err)
require.Len(t, messages, 1)
}
func TestDeliver_MalformedLinesSkipped(t *testing.T) {
// Fan-out is fire-and-forget: a malformed or message-less line is skipped (and logged), the
// remaining lines are still delivered
frag, err := marshalMessage(model.NewDefaultMessage("mytopic", "good"))
require.Nil(t, err)
body := []byte("this is not json\n{\"sender\":\"1.2.3.4\"}\n" + string(frag) + "\n\n")
messages, err := unmarshalMessageBody(body, 1<<20)
require.Nil(t, err)
require.Len(t, messages, 1)
require.Equal(t, "good", messages[0].Message)
}
// unmarshalMessageBody is a test helper collecting the messages of an NDJSON message body.
func unmarshalMessageBody(body []byte, maxLineBytes int) ([]*model.Message, error) {
var messages []*model.Message
err := decodeMessageBody(bytes.NewReader(body), maxLineBytes, func(m *model.Message) {
messages = append(messages, m)
})
return messages, err
}
func TestNop(t *testing.T) {
b, err := New(&Config{}, nil, nil) // not enabled -> nop cluster, no database required
require.Nil(t, err)
require.IsType(t, &nopCluster{}, b)
require.Nil(t, b.ForwardMessage(model.NewDefaultMessage("mytopic", "hi")))
// A single node is trivially the leader, so leader-gated jobs run without special-casing
require.True(t, b.IsLeader())
require.True(t, b.Healthy())
rr := httptest.NewRecorder()
b.ServeHTTP(rr, httptest.NewRequest("POST", MessagePath, nil))
require.Equal(t, 404, rr.Code)
require.Nil(t, b.Close())
}
func TestNew_EnabledRequiresDatabase(t *testing.T) {
_, err := New(&Config{Enabled: true, Secret: "secret"}, nil, nil)
require.Error(t, err)
require.Contains(t, err.Error(), "database")
}
func TestNew_Role(t *testing.T) {
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
conf := newTestMeshConfig("node-a", "http://127.0.0.1:1")
conf.Role = ""
c, err := New(conf, pool, nil)
require.Nil(t, err)
defer c.Close()
require.Equal(t, RoleActive, c.Members()[0].Role) // Unset means active, like a node that predates roles
// A typo must not silently make a node half passive (never leading, but still fencing)
conf = newTestMeshConfig("node-b", "http://127.0.0.1:1")
conf.Role = "pasive"
_, err = New(conf, pool, nil)
require.Error(t, err)
require.Contains(t, err.Error(), "experimental-cluster-role")
}
func TestDeliver_LineLimitBelowTheScanBufferIsEnforced(t *testing.T) {
// The configured limit is what the sender's own message size limit implies (MessageSizeLimit
// * 4 + 1024, so about 17KB for ntfy's default), which is well under bufio's buffer. A
// scanner whose initial buffer is bigger than the limit accepts tokens up to the buffer
// instead, so the limit silently did not apply to anything smaller than 64KB.
const limit = 512
big := model.NewDefaultMessage("mytopic", strings.Repeat("x", 4*limit))
line, err := marshalMessage(big)
require.Nil(t, err)
require.Greater(t, len(line), limit) // The line under test really is over the limit
var delivered []*model.Message
err = decodeMessageBody(bytes.NewReader(line), limit, func(m *model.Message) {
delivered = append(delivered, m)
})
require.Error(t, err)
require.Empty(t, delivered, "a line over the configured limit was delivered")
}
func TestDeliver_LineOfExactlyTheLimitIsAccepted(t *testing.T) {
// maxLineBytes is inclusive: bufio's own limit is exclusive, so this is off by one unless
// the decoder accounts for it, and a peer sending a message right at the limit would have
// its whole batch rejected
line, err := marshalMessage(model.NewDefaultMessage("mytopic", "hello"))
require.Nil(t, err)
limit := len(bytes.TrimRight(line, "\n"))
var delivered []*model.Message
collect := func(m *model.Message) { delivered = append(delivered, m) }
require.Nil(t, decodeMessageBody(bytes.NewReader(line), limit, collect))
require.Len(t, delivered, 1)
// One byte over, and the batch fails rather than delivering a line past the limit
delivered = nil
require.Error(t, decodeMessageBody(bytes.NewReader(line), limit-1, collect))
require.Empty(t, delivered)
}
// TestMesh_Soak floods the mesh with concurrent publishers and asserts exact delivery: every
// message reaches the peer exactly once, nothing is dropped, and batching keeps the request
// count far below the message count. Skipped unless NTFY_TEST_SOAK is set (it takes a few
// seconds and is meant for pre-deploy verification, not the regular suite).
func TestMesh_Soak(t *testing.T) {
if os.Getenv("NTFY_TEST_SOAK") == "" {
t.Skip("NTFY_TEST_SOAK not set")
}
// ~1000 msg/s aggregate (10x the ntfy.sh peak of ~88 msg/s): each publisher paces itself to
// 100 msg/s. Unthrottled publishing intentionally overruns the bounded per-peer queue (load
// shedding by design), so a zero-drop assertion only holds below the drain ceiling.
const (
publishers = 10
messagesPerPublisher = 300
publishInterval = 10 * time.Millisecond
total = publishers * messagesPerPublisher
)
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
var mu sync.Mutex
received := make(map[string]int, total) // message body -> count, to catch duplicates
requests := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
require.Nil(t, err)
messages, err := unmarshalMessageBody(body, 1<<20)
require.Nil(t, err)
mu.Lock()
requests++
for _, m := range messages {
received[m.Message]++
}
mu.Unlock()
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
conf := newTestMeshConfig("node-a", "http://127.0.0.1:1")
conf.BatchLinger = 50 * time.Millisecond
conf.NodeTTL = time.Minute // The fake peer never heartbeats; liveness is not under test here
registerFakePeer(t, pool, "node-peer", srv.URL)
mesh, err := newMeshCluster(conf, pool, nil)
require.Nil(t, err)
defer mesh.Close()
start := time.Now()
var wg sync.WaitGroup
for p := 0; p < publishers; p++ {
wg.Add(1)
go func(p int) {
defer wg.Done()
ticker := time.NewTicker(publishInterval)
defer ticker.Stop()
for i := 0; i < messagesPerPublisher; i++ {
require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", fmt.Sprintf("p%d-m%d", p, i))))
<-ticker.C
}
}(p)
}
wg.Wait()
waitFor(t, func() bool {
mu.Lock()
defer mu.Unlock()
return len(received) == total
})
elapsed := time.Since(start)
mu.Lock()
defer mu.Unlock()
for body, count := range received {
require.Equalf(t, 1, count, "message %s delivered %d times", body, count)
}
require.Less(t, requests, total/10, "expected strong batching under load")
t.Logf("soak: %d messages, %d requests (%.1f msgs/request), %.0f msgs/s",
total, requests, float64(total)/float64(requests), float64(total)/elapsed.Seconds())
}
// BenchmarkForwardMessage measures the publish-path cost of ForwardMessage: marshal + peer lookup (cached)
// + enqueue. The peer never drains, so enqueued fragments are dropped once the queue fills;
// the benchmark measures the hot path, not HTTP delivery.
func BenchmarkForwardMessage(b *testing.B) {
if os.Getenv("NTFY_TEST_DATABASE_URL") == "" {
b.Skip("NTFY_TEST_DATABASE_URL not set")
}
schemaDSN := dbtest.CreateTestPostgresSchema(b)
pool := openTestPool(b, schemaDSN)
conf := newTestMeshConfig("node-a", "http://127.0.0.1:1")
conf.BatchLinger = time.Minute // Never flush; we measure enqueue only
mesh, err := newMeshCluster(conf, pool, nil)
require.Nil(b, err)
defer mesh.Close()
registerFakePeer(b, pool, "node-peer", "http://127.0.0.1:1")
m := model.NewDefaultMessage("mytopic", "benchmark message body of typical size for a push")
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := mesh.ForwardMessage(m); err != nil {
b.Fatal(err)
}
}
}
// BenchmarkDecodeFanout measures the receive-path cost of decoding a 100-message NDJSON body.
func BenchmarkDecodeFanout(b *testing.B) {
frags := make([][]byte, 100)
for i := range frags {
data, err := marshalMessage(model.NewDefaultMessage("mytopic", fmt.Sprintf("benchmark message %d", i)))
require.Nil(b, err)
frags[i] = data
}
body := assembleMessageBody(frags)
b.SetBytes(int64(len(body)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
messages, err := unmarshalMessageBody(body, 1<<20)
if err != nil || len(messages) != 100 {
b.Fatal("decode failed")
}
}
}
cluster/registry/registry.go
+350
View File
@@ -0,0 +1,350 @@
// Package registry implements cluster membership and, derived from it, leadership for singleton
// jobs. Each node upserts its own row into the node_registry table with a fresh heartbeat and
// discovers its peers by reading the other fresh rows; the leader is simply the live active node
// with the lowest id (see IsLeader), so there is no second mechanism and no extra state.
//
// Nothing uses this yet: it is the membership half of multi-node ntfy, landed on its own so it
// can be reviewed without the fan-out that will sit on top of it. Node IDs are plain strings
// here; the cluster package will layer its own NodeID type over them.
package registry
import (
"context"
"fmt"
"sync"
"time"
"heckel.io/ntfy/v2/db"
"heckel.io/ntfy/v2/db/schema"
)
// Registry queries
const (
// Heartbeats are stamped with the database clock, never a node's: liveness compares them
// across nodes, and leadership is derived from liveness (see IsLeader)
upsertNodeQuery = `
INSERT INTO node_registry (node_id, advertise_url, role, last_heartbeat)
VALUES ($1, $2, $3, EXTRACT(EPOCH FROM now())::BIGINT)
ON CONFLICT (node_id) DO UPDATE SET advertise_url = EXCLUDED.advertise_url, role = EXCLUDED.role, last_heartbeat = EXCLUDED.last_heartbeat
`
// selectLiveNodesQuery returns every node that heartbeated within $1 seconds INCLUDING this
// one, with its heartbeat's age: the caller needs the whole set, because the leader is the
// live active node with the lowest id, and it filters the live ones from the recent ones
selectLiveNodesQuery = `SELECT node_id, advertise_url, role, EXTRACT(EPOCH FROM now())::BIGINT - last_heartbeat FROM node_registry WHERE last_heartbeat >= EXTRACT(EPOCH FROM now())::BIGINT - $1 ORDER BY node_id`
pruneStaleNodesQuery = `DELETE FROM node_registry WHERE last_heartbeat < EXTRACT(EPOCH FROM now())::BIGINT - $1`
deleteNodeQuery = `DELETE FROM node_registry WHERE node_id = $1`
)
// Node roles
const (
RoleActive Role = "active"
RolePassive Role = "passive"
)
// Schema version and queries
const (
maxOpTimeout = 5 * time.Second // Upper bound for one registry database call (ttl/2 below that)
// Leadership windows, as multiples of the node TTL. The hold-off is deliberately longer
// than the liveness window: the TTL also decides how fast a dead peer leaves the fan-out
// set and how fast an isolated node notices, so it has to stay short, while leadership
// would rather be slow and calm. The lease stays at one TTL, so belief ends exactly when
// peers stop counting this node as live.
//
// The invariant is leaseFactor < 1 + holdoffFactor: a successor cannot start its hold-off
// until the old leader's row is stale, one TTL after its last heartbeat, so the old belief
// always lapses first. The slack between the two, (1 + holdoffFactor - leaseFactor) TTLs,
// is how long a unit of work started under valid leadership is safe from a second leader.
// With a 30s TTL: lease 30s, hold-off 60s, 60s of slack, and up to ~90s with no leader
// after a crash.
leaseFactor = 1
holdoffFactor = 2
schemaStoreKey = "node_registry"
)
var (
// The role defaults to active, so rows upserted by nodes that predate roles (a rolling
// deploy) keep their old meaning. It comes last, where the migration adds it.
createTable = schema.AsMigrateFunc(fmt.Sprintf(`
CREATE TABLE IF NOT EXISTS node_registry (
node_id TEXT PRIMARY KEY,
advertise_url TEXT NOT NULL,
last_heartbeat BIGINT NOT NULL,
role TEXT NOT NULL DEFAULT '%s'
)
`, RoleActive))
// migrations maps a schema version to the migration upgrading it to the next version.
// Always append migrations at the end, never insert in the middle.
migrations = schema.Migrations{
{From: 1, To: 2}: schema.Query(fmt.Sprintf(`ALTER TABLE node_registry ADD COLUMN IF NOT EXISTS role TEXT NOT NULL DEFAULT '%s'`, RoleActive)),
}
)
// Role is a node's part in the cluster. An active node takes client traffic, may lead and may
// fence; a passive node (e.g. a hidden node under observation) does neither.
type Role string
// Valid reports whether r is one of the known roles
func (r Role) Valid() bool {
return r == RoleActive || r == RolePassive
}
// Peer is a live remote node as read from the registry.
type Peer struct {
NodeID string
AdvertiseURL string
Role Role
HeartbeatAt time.Time // When the peer last heartbeated, on this node's clock (database clock age at read time)
}
// Peers is a list of live peers, as returned by Peers and Refresh
type Peers []*Peer
// HasActive reports whether any of the peers is active, i.e. can take clients
func (p Peers) HasActive() bool {
for _, peer := range p {
if peer.Role == RoleActive {
return true
}
}
return false
}
// Registry is the node membership table (control plane): each node upserts its own row with a
// fresh heartbeat every few seconds, and peers are the other rows with a heartbeat newer than
// the TTL. Stale rows are pruned by the leader.
//
// Reads are split in two: Refresh queries the table and is called from the heartbeat loop, and
// Peers only ever returns the snapshot of the last Refresh. The publish path therefore never
// waits on the database, not even during an outage, when the snapshot simply goes stale.
//
// Leadership is derived from the same data, with no second mechanism: the leader is the live
// active node with the lowest id (see IsLeader).
type Registry struct {
pool *db.DB
nodeID string
advertiseURL string
role Role
ttl time.Duration
peers Peers // Snapshot of the last Refresh; nil until the first one
registeredAt time.Time // When the last successful Register was ISSUED; zero = never
refreshedAt time.Time // When the last successful Refresh was ISSUED; zero = never
lowestSince time.Time // When this node first saw itself as the lowest live active id; zero = it is not
mu sync.Mutex // Protects role, peers, registeredAt, refreshedAt and lowestSince
}
// New creates or migrates the registry schema and returns this node's membership handle. It
// does NOT register the node: joining the cluster is an explicit Register call, owned by the
// caller, so read-only uses of the registry stay side-effect free.
func New(pool *db.DB, nodeID, advertiseURL string, role Role, ttl time.Duration) (*Registry, error) {
if err := schema.Migrate(pool.Primary(), schema.Postgres, schemaStoreKey, createTable, migrations); err != nil {
return nil, err
}
return &Registry{
pool: pool,
nodeID: nodeID,
advertiseURL: advertiseURL,
role: role,
ttl: ttl,
}, nil
}
// SetRole changes the role this node registers with, at runtime (a draining active node acts as
// passive). Leadership ends at once on a step down to passive; peers see the new role
// at the next Register.
func (r *Registry) SetRole(role Role) {
r.mu.Lock()
defer r.mu.Unlock()
r.role = role
}
// Register upserts this node into the registry with a fresh heartbeat. The registration is
// dated from when the statement was ISSUED, not when it returned: that timestamp bounds how
// long this node may believe it is the leader, and a late response must not extend it.
func (r *Registry) Register() error {
ctx, cancel := r.opContext()
defer cancel()
issued := time.Now()
r.mu.Lock()
role := r.role
r.mu.Unlock()
if _, err := r.pool.ExecContext(ctx, upsertNodeQuery, r.nodeID, r.advertiseURL, role); err != nil {
return err
}
r.mu.Lock()
r.registeredAt = issued
r.mu.Unlock()
return nil
}
// Peers returns the live peer nodes as of the last Refresh (all registry rows with a fresh
// heartbeat then, excluding this node). It never touches the database: before the first Refresh
// there are no peers, and during an outage the snapshot is served as is (dead peers in it only
// cost failed sends).
func (r *Registry) Peers() Peers {
r.mu.Lock()
defer r.mu.Unlock()
return r.peers
}
// Fresh reports whether the snapshot Peers serves is recent enough to act on. Peers keeps
// serving the last known list when the database is unreachable, which is what keeps fan-out
// alive through a hiccup, but a view that old must not be reported outwards as the current
// membership (see cluster.Member).
func (r *Registry) Fresh() bool {
r.mu.Lock()
defer r.mu.Unlock()
return !r.refreshedAt.IsZero() && time.Since(r.refreshedAt) < r.ttl
}
// Prune deletes registry rows whose heartbeat is long expired. Only the leader calls this. It
// keeps recently live rows (see recent): a new node holds off for those, and must still see them.
func (r *Registry) Prune() error {
ctx, cancel := r.opContext()
defer cancel()
_, err := r.pool.ExecContext(ctx, pruneStaleNodesQuery, int64(r.recent().Seconds()))
return err
}
// IsLeader reports whether this node should run the cluster's singleton jobs. Leadership is not
// a separate mechanism: the leader is the live active node with the lowest id, which every node
// can work out from the membership data it already reads. A passive node never leads, and with
// no active node live there is no leader.
//
// Three conditions, and the last two are what keep two nodes from both believing:
//
// 1. This node is the lowest live active id, as of its last Refresh.
// 2. Its own registration is fresh, dated from when the heartbeat was issued (lease). A node
// that cannot reach the database stops believing within the lease, and it stops before any
// successor begins, because the successor cannot finish its hold-off that early.
// 3. It has been the lowest live active id, *continuously observed*, for the hold-off. A returning
// lower-id node would otherwise believe at once, while the incumbent still believes until
// its next read.
//
// "Continuously observed" is load-bearing. A process that was frozen (SIGSTOP, a paused VM, a
// long stall) resumes holding an old observation, and counting that pause toward the hold-off
// let it believe again the moment it registered once, while the incumbent had not yet noticed
// it was back. Refresh therefore restarts the hold-off whenever observation lapsed, and belief
// additionally requires a recent observation.
//
// Leadership is not a fence, and the slack above is not a promise. A caller must re-check
// IsLeader for each unit of work: a leader-gated job that blocks inside a database call can sit
// there past its own lease, and no window can bound that (a hung connection holds for minutes).
// Measured on three hosts with the database partitioned away from the leader: an unbounded call
// inside the leader-gated section kept a node acting as leader for 14.3s after its lease had
// expired, 8s of that alongside the new leader. Anything whose double execution matters needs
// its own claim row or has to be idempotent; leadership only keeps N nodes from doing the same
// idempotent work at once.
func (r *Registry) IsLeader() bool {
r.mu.Lock()
defer r.mu.Unlock()
if r.role != RoleActive {
return false // A passive node never leads; checked here too, since its row may still say active
} else if r.registeredAt.IsZero() || time.Since(r.registeredAt) >= r.lease() {
return false // Our registration is stale: peers no longer count us as live
}
if r.refreshedAt.IsZero() || time.Since(r.refreshedAt) >= r.lease() {
return false // We have not looked at the registry recently enough to trust what we saw
}
return !r.lowestSince.IsZero() && time.Since(r.lowestSince) >= r.holdoff()
}
// lease is how long this node may believe it leads on one successful heartbeat
func (r *Registry) lease() time.Duration {
return leaseFactor * r.ttl
}
// holdoff is how long this node must have been the lowest live active id before it believes
func (r *Registry) holdoff() time.Duration {
return holdoffFactor * r.ttl
}
// recent is how long after its last heartbeat a node may still believe it leads: its lease, plus
// the hold-off it waited before it led
func (r *Registry) recent() time.Duration {
return r.lease() + r.holdoff()
}
// Deregister deletes this node's registry row; called on shutdown.
func (r *Registry) Deregister() error {
ctx, cancel := r.opContext()
defer cancel()
_, err := r.pool.ExecContext(ctx, deleteNodeQuery, r.nodeID)
return err
}
// Refresh reads the live node set from the database, replaces the snapshot Peers serves, and
// updates this node's leadership standing. The mesh calls it once at startup and on every
// heartbeat tick, so both are bounded by the heartbeat interval. On error the previous snapshot
// and standing stay in place; leadership then lapses on its own via the registration deadline.
func (r *Registry) Refresh() (Peers, error) {
issued := time.Now()
peers, lowest, otherActive, err := r.queryLiveNodes()
if err != nil {
return nil, err
}
r.mu.Lock()
r.peers = peers
// A gap in observation does not count toward the hold-off: see IsLeader
lapsed := r.refreshedAt.IsZero() || issued.Sub(r.refreshedAt) >= r.lease()
if lowest != r.nodeID {
r.lowestSince = time.Time{}
} else if (r.lowestSince.IsZero() || lapsed) && !otherActive {
r.lowestSince = issued.Add(-r.holdoff()) // No other active node was live recently, so none can still believe it leads
} else if r.lowestSince.IsZero() || lapsed {
r.lowestSince = issued // Start, or restart, the promotion hold-off
}
r.refreshedAt = issued
r.mu.Unlock()
return peers, nil
}
// queryLiveNodes reads the live nodes and returns this node's peers, the lowest live active id
// (which may be this node, and is empty if no active node is live), and whether another active
// node was live recently: within the lease plus the hold-off, the longest that node may still
// believe it leads after its last heartbeat. Rows that recent but not live are not peers.
func (r *Registry) queryLiveNodes() (Peers, string, bool, error) {
ctx, cancel := r.opContext()
defer cancel()
rows, err := r.pool.QueryContext(ctx, selectLiveNodesQuery, int64(r.recent().Seconds()))
if err != nil {
return nil, "", false, err
}
defer rows.Close()
peers := make(Peers, 0)
lowest := ""
otherActive := false
for rows.Next() { // Ordered by node_id, so the first live active row is the lowest
p := &Peer{}
var age int64
if err := rows.Scan(&p.NodeID, &p.AdvertiseURL, &p.Role, &age); err != nil {
return nil, "", false, err
}
p.HeartbeatAt = time.Now().Add(-time.Duration(age) * time.Second)
if p.NodeID != r.nodeID && p.Role == RoleActive {
otherActive = true
}
if age > int64(r.ttl.Seconds()) {
continue // Recently live, not live: not a peer, and not a candidate
}
if lowest == "" && p.Role == RoleActive {
lowest = p.NodeID // A passive node never leads, so it does not count here
}
if p.NodeID != r.nodeID {
peers = append(peers, p)
}
}
if err := rows.Err(); err != nil {
return nil, "", false, err
}
return peers, lowest, otherActive, nil
}
// opContext bounds a registry database call, so a partition (dropped packets) makes it fail
// instead of hanging the heartbeat loop and shutdown
func (r *Registry) opContext() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), min(r.ttl/2, maxOpTimeout))
}
cluster/registry/registry_test.go
+697
View File
@@ -0,0 +1,697 @@
package registry
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/db"
"heckel.io/ntfy/v2/db/pg"
dbtest "heckel.io/ntfy/v2/db/test"
)
func openTestPool(t *testing.T, dsn string) *db.DB {
t.Helper()
host, err := pg.Open(dsn)
require.Nil(t, err)
d := db.New(host, nil)
t.Cleanup(func() { d.Close() })
return d
}
func TestRegistry_NewDoesNotRegister(t *testing.T) {
// New only sets up the schema and the identity handle; joining the cluster is an explicit
// Register call, owned by the caller (the mesh registers synchronously at construction).
// This keeps read-only uses (ops tooling, future admin endpoints) side-effect free.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
r1, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Equal(t, 0, countRows(t, pool, "node-1"))
require.Nil(t, r1.Register())
require.Equal(t, 1, countRows(t, pool, "node-1"))
}
func TestRegistry_RegisterAndPeers(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
r1, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Nil(t, r1.Register())
r2, err := New(pool, "node-2", "http://10.0.0.2:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Nil(t, r2.Register())
// Each node sees the other, never itself
peers, err := r1.Refresh()
require.Nil(t, err)
require.Len(t, peers, 1)
require.Equal(t, "node-2", peers[0].NodeID)
require.Equal(t, "http://10.0.0.2:2587", peers[0].AdvertiseURL)
peers, err = r2.Refresh()
require.Nil(t, err)
require.Len(t, peers, 1)
require.Equal(t, "node-1", peers[0].NodeID)
}
func TestRegistry_ReRegisterUpdatesAdvertiseURL(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
r1, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
// The same node comes back under a new address; the upsert replaces the row
old, err := New(pool, "node-2", "http://old:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Nil(t, old.Register())
renewed, err := New(pool, "node-2", "http://new:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Nil(t, renewed.Register())
peers, err := r1.Refresh()
require.Nil(t, err)
require.Len(t, peers, 1)
require.Equal(t, "http://new:2587", peers[0].AdvertiseURL)
}
func TestRegistry_PeersIsTheLastRefresh(t *testing.T) {
// Peers never queries: it is empty before the first Refresh, and a node joining afterwards
// stays invisible until the next one (the heartbeat's job)
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
r1, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Empty(t, r1.Peers())
r2, err := New(pool, "node-2", "http://10.0.0.2:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Nil(t, r2.Register())
require.Empty(t, r1.Peers())
_, err = r1.Refresh()
require.Nil(t, err)
require.Len(t, r1.Peers(), 1)
}
func TestRegistry_TTLExcludesSilentNodes(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
r1, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
// A node whose heartbeat is older than the TTL does not count as live
insertNodeAt(t, pool, "node-silent", "http://10.0.0.9:2587", time.Now().Add(-2*time.Minute))
peers, err := r1.Refresh()
require.Nil(t, err)
require.Empty(t, peers)
}
func TestRegistry_PruneDeletesLongDeadOnly(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
r1, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
// One node beyond the 3x TTL grace period, one merely stale
insertNodeAt(t, pool, "node-long-dead", "http://10.0.0.8:2587", time.Now().Add(-4*time.Minute))
insertNodeAt(t, pool, "node-slow", "http://10.0.0.9:2587", time.Now().Add(-2*time.Minute))
require.Nil(t, r1.Prune())
require.Equal(t, 0, countRows(t, pool, "node-long-dead"))
require.Equal(t, 1, countRows(t, pool, "node-slow")) // Slow, not dead: kept
}
func TestRegistry_Deregister(t *testing.T) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
r1, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Nil(t, r1.Register())
require.Equal(t, 1, countRows(t, pool, "node-1"))
require.Nil(t, r1.Deregister())
require.Equal(t, 0, countRows(t, pool, "node-1"))
}
func TestRegistry_PeersSurvivesDatabaseOutage(t *testing.T) {
// During a database outage, Refresh fails but Peers keeps serving the last snapshot, without
// a query and without an error: fan-out keeps flowing to known peers, and the publish path
// neither waits on the database nor logs a warning per message.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
r1, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
r2, err := New(pool, "node-2", "http://10.0.0.2:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Nil(t, r2.Register())
_, err = r1.Refresh()
require.Nil(t, err)
require.Nil(t, pool.Close())
_, err = r1.Refresh()
require.NotNil(t, err)
started := time.Now()
peers := r1.Peers()
require.Less(t, time.Since(started), 100*time.Millisecond) // No database round trip, let alone a timeout
require.Len(t, peers, 1)
require.Equal(t, "node-2", peers[0].NodeID)
}
func TestRegistry_ConcurrentCreate(t *testing.T) {
// Multiple nodes cold-booting on a fresh database must not race on table creation: CREATE
// TABLE IF NOT EXISTS is not atomic in PostgreSQL, so creation is serialized via an advisory
// lock. Without it, this test fails sporadically with a duplicate-key error on pg_class.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
const n = 8
errs := make(chan error, n)
for i := 0; i < n; i++ {
go func(i int) {
pool, err := pg.Open(schemaDSN)
if err != nil {
errs <- err
return
}
defer pool.DB.Close()
_, err = New(db.New(pool, nil), fmt.Sprintf("node-%d", i), "http://127.0.0.1:1", RoleActive, time.Second)
errs <- err
}(i)
}
for i := 0; i < n; i++ {
require.Nil(t, <-errs)
}
}
func TestRegistry_SchemaVersionWritten(t *testing.T) {
// The registry participates in the shared schema_version framework like every other store,
// so future table changes can be applied as migrations.
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
_, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
var version int
require.Nil(t, pool.QueryRow(`SELECT version FROM schema_version WHERE store = $1`, schemaStoreKey).Scan(&version))
require.Equal(t, 2, version)
// Setup is idempotent: a second node boots against the migrated schema
_, err = New(pool, "node-2", "http://10.0.0.2:2587", RoleActive, time.Minute)
require.Nil(t, err)
}
func TestRegistry_OlderNodeStartsOnNewerSchema(t *testing.T) {
// During a rolling upgrade, a node still running older code shares a schema migrated by a
// newer node: it starts (migrations are additive, see package schema) and leaves the version alone
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
_, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
_, err = pool.Exec(`UPDATE schema_version SET version = 99 WHERE store = $1`, schemaStoreKey)
require.Nil(t, err)
_, err = New(pool, "node-2", "http://10.0.0.2:2587", RoleActive, time.Minute)
require.Nil(t, err)
var version int
require.Nil(t, pool.QueryRow(`SELECT version FROM schema_version WHERE store = $1`, schemaStoreKey).Scan(&version))
require.Equal(t, 99, version)
}
// insertNodeAt registers a fake node with a chosen heartbeat. Register() stamps the database
// clock, which is what production wants but leaves tests no way to age a row.
func insertNodeAt(t *testing.T, pool *db.DB, nodeID, url string, heartbeat time.Time) {
t.Helper()
_, err := pool.Exec(`INSERT INTO node_registry (node_id, advertise_url, last_heartbeat) VALUES ($1, $2, $3)`, nodeID, url, heartbeat.Unix())
require.Nil(t, err)
}
func countRows(t *testing.T, pool *db.DB, nodeID string) int {
t.Helper()
var count int
require.Nil(t, pool.QueryRow(`SELECT COUNT(*) FROM node_registry WHERE node_id = $1`, nodeID).Scan(&count))
return count
}
func TestRegistry_RefreshReplacesSnapshot(t *testing.T) {
// Refresh (called by the mesh heartbeat) sees a newly joined node and replaces the snapshot
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
r1, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Nil(t, r1.Register())
_, err = r1.Refresh()
require.Nil(t, err)
require.Len(t, r1.Peers(), 0) // Snapshot taken while alone
r2, err := New(pool, "node-2", "http://10.0.0.2:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Nil(t, r2.Register())
require.Len(t, r1.Peers(), 0) // Still the old snapshot
peers, err := r1.Refresh()
require.Nil(t, err)
require.Len(t, peers, 1)
require.Equal(t, "node-2", peers[0].NodeID)
require.Len(t, r1.Peers(), 1) // Snapshot replaced
}
// Leadership is derived from membership, so these tests drive Register/Refresh by hand (the
// mesh's heartbeat does both every tick) and assert the one invariant that matters: never two
// leaders, and eventually one.
// leaderTestTTL is the liveness window and the promotion hold-off. Registry calls are bounded
// at ttl/2 (see opContext), so a tighter value makes these tests fail on a slow or contended
// database rather than on the behaviour they are checking.
const (
leaderTestTTL = time.Second
// Everything leadership does is a multiple of the TTL (lease, hold-off), and a takeover
// costs TTL + hold-off, so the waits are derived rather than guessed
leaderTestLease = leaseFactor * leaderTestTTL
leaderTestHoldoff = holdoffFactor * leaderTestTTL
leaderTestWait = 3 * (leaderTestTTL + leaderTestHoldoff)
// How often the leadership tests drive a tick. Each tick is a round trip per registry, so
// this is also the query rate they put on the database while running in parallel; the
// windows under test are seconds, so 50ms granularity costs nothing.
leaderTestPoll = 50 * time.Millisecond
)
func tick(t *testing.T, rs ...*Registry) {
t.Helper()
for _, r := range rs {
require.Nil(t, r.Register())
_, err := r.Refresh()
require.Nil(t, err)
}
}
func waitForLeader(t *testing.T, r *Registry, others ...*Registry) {
t.Helper()
deadline := time.Now().Add(leaderTestWait)
for time.Now().Before(deadline) {
tick(t, append([]*Registry{r}, others...)...)
for _, o := range others {
require.False(t, r.IsLeader() && o.IsLeader(), "two leaders at once")
}
if r.IsLeader() {
return
}
time.Sleep(leaderTestPoll)
}
t.Fatal("never became leader")
}
func TestRegistry_LowestLiveNodeLeads(t *testing.T) {
t.Parallel() // Own schema and own registries, so these run alongside each other
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
a, err := New(pool, "app1", "http://10.0.0.1:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
b, err := New(pool, "app2", "http://10.0.0.2:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
c, err := New(pool, "app3", "http://10.0.0.3:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
// app2 and app3 register first; app1 sees them live and waits out the hold-off (app2, alone
// when it first ticked, may lead meanwhile, never at the same time)
tick(t, b, c, a)
require.False(t, a.IsLeader())
waitForLeader(t, a, b, c)
require.False(t, b.IsLeader())
require.False(t, c.IsLeader())
}
func TestRegistry_LeaderFailoverHasAGapNotAnOverlap(t *testing.T) {
t.Parallel() // Own schema and own registries, so these run alongside each other
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
a, err := New(pool, "app1", "http://10.0.0.1:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
b, err := New(pool, "app2", "http://10.0.0.2:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
waitForLeader(t, a, b)
// app1 stops heartbeating (crashed, or cut off from the database). Only app2 ticks now, so
// app1's belief must lapse on its own registration deadline before app2's hold-off ends.
started := time.Now()
for !b.IsLeader() {
require.False(t, a.IsLeader() && b.IsLeader(), "two leaders at once")
require.Less(t, time.Since(started), leaderTestWait, "app2 never took over")
tick(t, b)
time.Sleep(leaderTestPoll)
}
require.False(t, a.IsLeader())
t.Logf("takeover after the leader stopped heartbeating: %v (TTL %v)", time.Since(started).Round(time.Millisecond), leaderTestTTL)
}
func TestRegistry_ReturningLowerNodePreemptsWithoutOverlap(t *testing.T) {
// The case that makes this approach need a hold-off at all: app1 comes back and will take
// leadership from app2 purely because its id sorts lower. It must not believe until app2's
// own belief can no longer be alive.
t.Parallel() // Own schema and own registries, so these run alongside each other
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
b, err := New(pool, "app2", "http://10.0.0.2:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
waitForLeader(t, b)
a, err := New(pool, "app1", "http://10.0.0.1:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
tick(t, a)
require.False(t, a.IsLeader(), "a returning lower-id node must wait out the hold-off")
// Both keep ticking: at no point may both believe
started := time.Now()
for !a.IsLeader() {
require.False(t, a.IsLeader() && b.IsLeader(), "two leaders at once")
require.Less(t, time.Since(started), leaderTestWait, "app1 never took over")
tick(t, a, b)
time.Sleep(leaderTestPoll)
}
require.False(t, b.IsLeader(), "the incumbent must have stepped down")
t.Logf("preemption by a returning lower id took %v", time.Since(started).Round(time.Millisecond))
}
func TestRegistry_PartitionedLeaderStepsDown(t *testing.T) {
// A leader that cannot reach the database stops believing within one TTL, because its
// registration deadline runs out. Peers stop seeing it as live at the same point.
t.Parallel() // Own schema and own registries, so these run alongside each other
schemaDSN := dbtest.CreateTestPostgresSchema(t)
pool := openTestPool(t, schemaDSN)
a, err := New(pool, "app1", "http://10.0.0.1:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
waitForLeader(t, a)
require.Nil(t, pool.Close()) // The database is gone for this node
require.Error(t, a.Register())
require.Eventually(t, func() bool { return !a.IsLeader() }, 2*time.Second, 5*time.Millisecond)
}
func TestRegistry_LivenessIsDecidedByTheDatabaseClock(t *testing.T) {
// Liveness timestamps are compared across nodes, so both the write and the cutoff have to
// come from the database: with node clocks, one running fast declares live peers dead and
// one running behind looks dead to everybody. Behaviour alone cannot tell the two apart
// here, because the test process and the database share a clock, so this pins the mechanism
// as well. (The cross-host version is a harness drill: a node's clock pushed an hour ahead
// changed neither liveness nor leadership.)
require.Contains(t, upsertNodeQuery, "EXTRACT(EPOCH FROM now())", "heartbeats must be stamped by the database")
require.Contains(t, selectLiveNodesQuery, "EXTRACT(EPOCH FROM now())", "the liveness cutoff must be evaluated by the database")
require.Contains(t, pruneStaleNodesQuery, "EXTRACT(EPOCH FROM now())", "pruning must use the database clock")
// And the behaviour that follows: this node's heartbeat lands on the database's clock, a row
// as an hour-fast node would have written it still counts as live, and one from an hour-slow
// node does not.
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
r, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Nil(t, r.Register())
var skew int64
require.Nil(t, pool.QueryRow(`SELECT EXTRACT(EPOCH FROM now())::BIGINT - last_heartbeat FROM node_registry WHERE node_id = 'node-1'`).Scan(&skew))
require.LessOrEqual(t, skew, int64(1))
insertNodeAt(t, pool, "node-fast", "http://10.0.0.2:2587", time.Now().Add(time.Hour))
insertNodeAt(t, pool, "node-slow", "http://10.0.0.3:2587", time.Now().Add(-time.Hour))
peers, err := r.Refresh()
require.Nil(t, err)
require.Len(t, peers, 1)
require.Equal(t, "node-fast", peers[0].NodeID)
}
func TestRegistry_HeartbeatWithoutObservingIsNotLeadership(t *testing.T) {
// A node can keep its registration fresh while its view of the registry goes stale: its
// refresh stalls, or it only ever writes. It must stop believing then, because the hold-off
// it earned rests on an observation that no longer holds. Without that check the incumbent
// keeps leading while a returning lower-id node earns its own hold-off, and both believe.
t.Parallel() // Own schema and own registries, so these run alongside each other
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
b, err := New(pool, "app2", "http://10.0.0.2:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
waitForLeader(t, b)
a, err := New(pool, "app1", "http://10.0.0.1:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
deadline := time.Now().Add(leaderTestWait)
for time.Now().Before(deadline) {
require.Nil(t, b.Register()) // The incumbent's registration stays fresh...
tick(t, a) // ...but only app1 ever looks at the registry
require.False(t, a.IsLeader() && b.IsLeader(), "two leaders at once")
if a.IsLeader() {
require.False(t, b.IsLeader(), "the incumbent kept believing on a stale observation")
return
}
time.Sleep(leaderTestPoll)
}
t.Fatal("app1 never took over")
}
func TestRegistry_FrozenLeaderMustReEarnTheHoldOff(t *testing.T) {
// A process that was frozen (SIGSTOP, a paused VM, a long stall) comes back holding an old
// observation. Counting that pause toward the promotion hold-off let it believe again the
// moment it registered once, while the incumbent had not yet noticed it was back: two
// leaders, measured at 436ms in a two-process run before this was fixed.
t.Parallel() // Own schema and own registries, so these run alongside each other
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
a, err := New(pool, "app1", "http://10.0.0.1:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
b, err := New(pool, "app2", "http://10.0.0.2:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
waitForLeader(t, a, b)
// app1 freezes: no heartbeats, no reads. Its own deadlines lapse and app2 takes over,
// which means ticking only app2 from here (ticking app1 would thaw it).
time.Sleep(2 * leaderTestLease)
require.False(t, a.IsLeader())
frozen := time.Now()
for !b.IsLeader() {
require.False(t, a.IsLeader() && b.IsLeader(), "two leaders at once")
require.Less(t, time.Since(frozen), leaderTestWait, "app2 never took over")
tick(t, b)
time.Sleep(leaderTestPoll)
}
// app1 thaws. One register plus one refresh must NOT hand leadership straight back.
require.Nil(t, a.Register())
_, err = a.Refresh()
require.Nil(t, err)
require.False(t, a.IsLeader(), "a thawed node believed again without re-earning the hold-off")
require.True(t, b.IsLeader(), "the incumbent should still hold it at this point")
// And once it has observed continuously for a hold-off, it may take over, with b stepping
// down first (that part is TestRegistry_ReturningLowerNodePreemptsWithoutOverlap)
started := time.Now()
for !a.IsLeader() {
require.False(t, a.IsLeader() && b.IsLeader(), "two leaders at once")
require.Less(t, time.Since(started), leaderTestWait, "app1 never recovered leadership")
tick(t, a, b)
time.Sleep(leaderTestPoll)
}
}
func TestRegistry_LeaseLapsesBeforeAnySuccessorBegins(t *testing.T) {
// The invariant behind "never two leaders": a deposed leader's belief must end before any
// successor can finish its hold-off. The successor cannot even start counting until the old
// leader's row is stale, one TTL after its last heartbeat, so the comparison is
// lease < TTL + hold-off. Asserted here because the factors are tunable.
require.Less(t, leaseFactor, 1+holdoffFactor,
"a leader could still believe while a successor starts: lease %d TTLs, successor starts at %d TTLs",
leaseFactor, 1+holdoffFactor)
// And belief must not outlive liveness by more than the hold-off can absorb, nor be so
// short that it ends before peers would even notice this node is gone
require.GreaterOrEqual(t, leaseFactor, 1, "the lease must cover the liveness window")
}
func TestRegistry_ViewIsOnlyFreshAfterARecentRefresh(t *testing.T) {
// Peers serves the last known list when the database is unreachable, which keeps fan-out
// alive but must not be mistaken for the current membership: anything that reports
// membership outwards (the member list the LB agents read) needs to know the difference.
t.Parallel()
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
r, err := New(pool, "node-a", "http://127.0.0.1:1", RoleActive, leaderTestTTL)
require.Nil(t, err)
require.False(t, r.Fresh(), "a registry that has never read the database is not fresh")
require.Nil(t, r.Register())
_, err = r.Refresh()
require.Nil(t, err)
require.True(t, r.Fresh())
// The view ages out on its own, without any failed call
deadline := time.Now().Add(leaderTestWait)
for r.Fresh() && time.Now().Before(deadline) {
time.Sleep(leaderTestPoll)
}
require.False(t, r.Fresh(), "the view never went stale")
}
func TestRegistry_RoleRoundTrips(t *testing.T) {
// Every node writes its role with each heartbeat, so peers read it from the snapshot they
// already take, and a node restarted with another role is seen with the new one
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
active, err := New(pool, "node-1", "http://10.0.0.1:2587", RoleActive, time.Minute)
require.Nil(t, err)
passive, err := New(pool, "node-2", "http://10.0.0.2:2587", RolePassive, time.Minute)
require.Nil(t, err)
tick(t, active, passive, active) // Again, so the first node's snapshot includes the second
require.Len(t, active.Peers(), 1)
require.Equal(t, RolePassive, active.Peers()[0].Role)
require.Len(t, passive.Peers(), 1)
require.Equal(t, RoleActive, passive.Peers()[0].Role)
// The same node comes back as active; the upsert replaces its role
restarted, err := New(pool, "node-2", "http://10.0.0.2:2587", RoleActive, time.Minute)
require.Nil(t, err)
require.Nil(t, restarted.Register())
peers, err := active.Refresh()
require.Nil(t, err)
require.Len(t, peers, 1)
require.Equal(t, RoleActive, peers[0].Role)
}
func TestRegistry_MigratesFromVersion1(t *testing.T) {
// Deployments that created the table before roles existed upgrade in place; their rows were
// all written by nodes that knew nothing of roles, so they read as active
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
_, err := pool.Exec(`CREATE TABLE node_registry (node_id TEXT PRIMARY KEY, advertise_url TEXT NOT NULL, last_heartbeat BIGINT NOT NULL)`)
require.Nil(t, err)
_, err = pool.Exec(`CREATE TABLE schema_version (store TEXT PRIMARY KEY, version INT NOT NULL)`)
require.Nil(t, err)
_, err = pool.Exec(`INSERT INTO schema_version (store, version) VALUES ($1, 1)`, schemaStoreKey)
require.Nil(t, err)
insertNodeAt(t, pool, "node-old", "http://10.0.0.9:2587", time.Now())
r, err := New(pool, "node-1", "http://10.0.0.1:2587", RolePassive, time.Minute)
require.Nil(t, err)
require.Nil(t, r.Register())
peers, err := r.Refresh()
require.Nil(t, err)
require.Len(t, peers, 1)
require.Equal(t, RoleActive, peers[0].Role)
var version int
require.Nil(t, pool.QueryRow(`SELECT version FROM schema_version WHERE store = $1`, schemaStoreKey).Scan(&version))
require.Equal(t, 2, version)
}
func TestRegistry_PassiveNodeNeverLeadsAlone(t *testing.T) {
// A passive node is not behind the load balancer, so the singleton jobs must not move there,
// not even when it is the only live node: no leader is better than a hidden one
t.Parallel()
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
a, err := New(pool, "app1", "http://10.0.0.1:2587", RolePassive, leaderTestTTL)
require.Nil(t, err)
tickPastHoldoff(t, func() {
require.False(t, a.IsLeader(), "a passive node believed it leads")
}, a)
}
func TestRegistry_NoLeaderWhenOnlyPassiveNodesAreLive(t *testing.T) {
t.Parallel()
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
a, err := New(pool, "app1", "http://10.0.0.1:2587", RolePassive, leaderTestTTL)
require.Nil(t, err)
b, err := New(pool, "app2", "http://10.0.0.2:2587", RolePassive, leaderTestTTL)
require.Nil(t, err)
tickPastHoldoff(t, func() {
require.False(t, a.IsLeader() || b.IsLeader(), "a passive node believed it leads")
}, a, b)
}
func TestRegistry_PassiveNodeNeverLeadsWithActivePeers(t *testing.T) {
// The passive node has the lowest id, which alone would make it the leader
t.Parallel()
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
a, err := New(pool, "app1", "http://10.0.0.1:2587", RolePassive, leaderTestTTL)
require.Nil(t, err)
b, err := New(pool, "app2", "http://10.0.0.2:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
tickPastHoldoff(t, func() {
require.False(t, a.IsLeader(), "a passive node believed it leads")
}, a, b)
require.True(t, b.IsLeader(), "the only active node never took over")
}
func TestRegistry_MixedRegistryLowestActiveLeads(t *testing.T) {
t.Parallel()
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
a, err := New(pool, "app1", "http://10.0.0.1:2587", RolePassive, leaderTestTTL)
require.Nil(t, err)
b, err := New(pool, "app2", "http://10.0.0.2:2587", RolePassive, leaderTestTTL)
require.Nil(t, err)
c, err := New(pool, "app3", "http://10.0.0.3:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
d, err := New(pool, "app4", "http://10.0.0.4:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
waitForLeader(t, c, a, b, d)
require.False(t, a.IsLeader())
require.False(t, b.IsLeader())
require.False(t, d.IsLeader())
}
// tickPastHoldoff drives the registries for longer than any takeover takes (liveness window
// plus hold-off, plus a TTL of margin), calling check after every tick
func tickPastHoldoff(t *testing.T, check func(), rs ...*Registry) {
t.Helper()
deadline := time.Now().Add(2*leaderTestTTL + leaderTestHoldoff)
for time.Now().Before(deadline) {
tick(t, rs...)
check()
time.Sleep(leaderTestPoll)
}
}
func TestRegistry_AloneLeadsRightAway(t *testing.T) {
// The hold-off keeps a previous leader's belief from overlapping with the next one. A node
// that sees no other active node, live or recently live, has nobody to overlap with: it
// leads at once, so a restarted single node does not pause its scheduled jobs for a minute
t.Parallel()
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
a, err := New(pool, "app1", "http://10.0.0.1:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
tick(t, a)
require.True(t, a.IsLeader())
}
func TestRegistry_OnlyPassivePeersLeadsRightAway(t *testing.T) {
t.Parallel()
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
b, err := New(pool, "app0", "http://10.0.0.2:2587", RolePassive, leaderTestTTL) // Lower id, but passive
require.Nil(t, err)
a, err := New(pool, "app1", "http://10.0.0.1:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
tick(t, b, a)
require.True(t, a.IsLeader())
require.False(t, b.IsLeader())
}
func TestRegistry_RecentlyLiveActivePeerMeansHoldOff(t *testing.T) {
// Another active node stopped heartbeating a moment ago: it may still believe it leads
t.Parallel()
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
a, err := New(pool, "app1", "http://10.0.0.1:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
insertNodeAt(t, pool, "app0", "http://10.0.0.9:2587", time.Now().Add(-2*leaderTestTTL)) // Stale, but recent
tick(t, a)
require.False(t, a.IsLeader())
waitForLeader(t, a)
}
func TestRegistry_PruneKeepsRecentlyLiveActiveNodes(t *testing.T) {
// A crashed leader's row must outlive the window in which a new node still holds off for it,
// or Prune would let the next node lead while the old one may still believe it does
t.Parallel()
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
a, err := New(pool, "app1", "http://10.0.0.1:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
insertNodeAt(t, pool, "app0", "http://10.0.0.9:2587", time.Now().Add(-a.lease()-a.holdoff()+leaderTestTTL))
require.Nil(t, a.Prune())
require.Equal(t, 1, countRows(t, pool, "app0"))
tick(t, a)
require.False(t, a.IsLeader())
}
func TestRegistry_SetRolePassiveHandsLeadershipOver(t *testing.T) {
// A draining node acts as passive at runtime: it stops leading at once, and its
// peers see the new role at its next heartbeat and take over
t.Parallel()
pool := openTestPool(t, dbtest.CreateTestPostgresSchema(t))
a, err := New(pool, "app1", "http://10.0.0.1:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
b, err := New(pool, "app2", "http://10.0.0.2:2587", RoleActive, leaderTestTTL)
require.Nil(t, err)
waitForLeader(t, a, b)
a.SetRole(RolePassive)
require.False(t, a.IsLeader())
tick(t, a, b)
peers := b.Peers()
require.Len(t, peers, 1)
require.Equal(t, RolePassive, peers[0].Role)
waitForLeader(t, b, a)
}
func TestPeers_HasActive(t *testing.T) {
// Whether there is somewhere for a node's clients to go: a passive peer takes none
require.False(t, Peers(nil).HasActive())
require.False(t, Peers{
&Peer{NodeID: "app2", Role: RolePassive},
}.HasActive())
require.True(t, Peers{
&Peer{NodeID: "app2", Role: RolePassive},
&Peer{NodeID: "app3", Role: RoleActive},
}.HasActive())
}
cluster/types.go
+150
View File
@@ -0,0 +1,150 @@
package cluster
import (
"time"
"heckel.io/ntfy/v2/model"
"heckel.io/ntfy/v2/util"
)
// Config configures the cluster. It is assembled by the server from its own config, which keeps
// this package free of server types.
type Config struct {
Enabled bool // Master switch; when false, New returns the nop cluster
NodeID NodeID // Stable per-node identifier; required
AdvertiseURL string // Base URL peers use to reach this node's fan-out endpoint
Role Role // Active (default) or passive: a passive node never leads and never fences
Secret string // Shared secret authenticating node-to-node fan-out requests
HeartbeatInterval time.Duration // How often the node registry heartbeat is refreshed
NodeTTL time.Duration // Registry rows older than this do not count as live peers
BatchLinger time.Duration // How long messages wait in a peer queue to form a batch; 0 = send immediately
MaxMessageBytes int64 // Upper bound for a single message on the wire (batch limits derive from this)
UsersDeletedFunc UsersDeletedFunc // Told that a peer deleted users; may be nil
AccessChangedFunc AccessChangedFunc // Told that a peer changed access entries; may be nil
TopicsAddedFunc TopicsAddedFunc // Told about topics that just gained their first subscriber on a peer; may be nil
IsolatedFunc IsolatedFunc // Called while this node is cut off from the cluster; may be nil
}
// DeliverFunc hands a message received from a peer node to this node's local subscribers. The
// server supplies it, which inverts the dependency: this package never imports the server.
type DeliverFunc func(m *model.Message)
// State is a state delta for Cluster.BroadcastState.
type State struct {
AddedTopics []string // Topics that just gained their first local subscriber on this node
DeletedUserIDs []string // Users this node just deleted: peers end their streams
AccessChanged bool // Access entries changed (written by this node): peers reload their access cache
}
// UsersDeletedFunc is called with the IDs of users a peer just deleted (State.DeletedUserIDs). The
// server supplies it to end those users' streams on this node; it must not re-broadcast (loop
// prevention). Access changes do not need this, since peers reload their access cache and end
// the streams that lost access themselves (see AccessChangedFunc), but a deleted user without
// access entries (e.g. an admin) leaves nothing in the cache to compare.
type UsersDeletedFunc func(userIDs []string)
// TopicsAddedFunc is called with topics a peer announced as having just gained their first
// subscriber there. The server supplies it to drop cached negative rate-visitor lookups for
// those topics; it must not re-broadcast (loop prevention). It is a cache hint only: a lost
// announcement costs a stale lookup until its TTL, never a lost message.
type TopicsAddedFunc func(topics []string)
// AccessChangedFunc is called when a peer changed access entries (State.AccessChanged). The server
// supplies it to reload its access cache, which ends the local streams that lost access; it must
// not re-broadcast (loop prevention). A lost signal is caught by the periodic access cache reload.
type AccessChangedFunc func()
// IsolatedFunc is called while this node lost its registration but a peer is healthy: peers no
// longer forward messages to it, so the server closes its subscribers, which reconnect to a
// healthy node (see meshCluster.maybeIsolated).
type IsolatedFunc func()
// Status is a node's view of the cluster, served on StatusPath. Times are unix seconds, 0 for never.
type Status struct {
NodeID NodeID `json:"node_id"`
Role Role `json:"role"`
Leader bool `json:"leader"`
Healthy bool `json:"healthy"`
Time int64 `json:"time"` // When this view was taken, to judge the other timestamps by
Peers []*PeerStatus `json:"peers"`
}
// PeerStatus is one peer as seen from this node: every live peer, plus any peer this node has
// exchanged batches with since it started (Live false once it left the registry)
type PeerStatus struct {
NodeID NodeID `json:"node_id"`
AdvertiseURL string `json:"advertise_url,omitempty"`
Role Role `json:"role,omitempty"`
Live bool `json:"live"`
HeartbeatAge int64 `json:"heartbeat_age,omitempty"` // Seconds since the peer's last registry heartbeat, as of the last refresh
LastSeen int64 `json:"last_seen,omitempty"` // When this node last saw the peer in the registry (or first heard from it)
Out PeerOutbound `json:"out"`
In PeerInbound `json:"in"`
}
// PeerOutbound counts the message batches this node sent to a peer since it started
type PeerOutbound struct {
BatchesOK int64 `json:"batches_ok"`
BatchesFailed int64 `json:"batches_failed"`
Messages int64 `json:"messages"` // Messages in batches the peer accepted
Dropped int64 `json:"dropped"` // Messages dropped because the peer's queue was full
Queued int `json:"queued"` // Messages waiting in the peer's queue right now
LastOK int64 `json:"last_ok,omitempty"`
LastError string `json:"last_error,omitempty"`
LastErrorAt int64 `json:"last_error_at,omitempty"`
LastFailed bool `json:"last_failed,omitempty"` // The most recent batch failed; timestamps cannot order outcomes within one second
}
// PeerInbound counts the message batches this node received from a peer since it started
type PeerInbound struct {
Batches int64 `json:"batches"`
Messages int64 `json:"messages"`
LastAt int64 `json:"last_at,omitempty"`
}
// peerStats counts the message batches exchanged with one peer, for Status. It outlives the peer's
// queue, so a peer that left still shows what happened before it did.
type peerStats struct {
out PeerOutbound // Queued is filled in when Status is built, from the live queue
in PeerInbound
lastSeen int64 // Unix seconds; a peer gone for peerStatsRetention is forgotten
}
// apiHealth is the answer on HealthPath: written by handleHealth, read by peerHealthy, so this
// package owns both ends of it. The status code carries the same answer for anything that only
// looks at that, but a peer reads the field, since a 200 from something that is not an ntfy
// cluster node (a proxy on a misconfigured advertise URL) says nothing.
type apiHealth struct {
Healthy bool `json:"healthy"`
}
// apiState is the peer state-exchange envelope. Each concern is an optional section; future
// concerns (rate limit counters, stats) become siblings of Topics.
type apiState struct {
Topics *apiStateTopics `json:"topics,omitempty"`
DeletedUserIDs []string `json:"deleted_user_ids,omitempty"`
AccessChanged bool `json:"access_changed,omitempty"`
}
// apiStateTopics carries topics that just gained their first subscriber on the sending node.
type apiStateTopics struct {
Added []string `json:"added,omitempty"`
}
// peerQueue is the bounded, batching send queue for a single peer, pinned to the advertise URL
// the peer was created with: a peer re-registering under a different advertise URL is treated
// as a replacement (reconcile retires the old queue; ForwardMessage creates a fresh one on demand).
type peerQueue struct {
advertiseURL string
queue *util.LingerQueue[[]byte]
}
// apiMessage is one line of a message request body (NDJSON: one message per line; a single
// message is just a one-line body). It carries the two fields that model.Message does not
// serialize to JSON (Sender and User), which are needed to reconstruct the visitor on the
// receiving node. The origin node travels in a request header, not in the body.
type apiMessage struct {
Sender string `json:"sender,omitempty"`
User string `json:"user,omitempty"`
Message *model.Message `json:"message"`
}
cluster/util.go
+63
View File
@@ -0,0 +1,63 @@
package cluster
import (
"bufio"
"bytes"
"encoding/json"
"io"
"net/netip"
"heckel.io/ntfy/v2/log"
"heckel.io/ntfy/v2/model"
)
// marshalMessage serializes one message and its non-JSON fields (Sender, User) as an
// apiMessage line. Lines are marshaled once per publish and shared across all per-peer
// queues; assembleMessageBody joins them without re-marshaling.
func marshalMessage(m *model.Message) ([]byte, error) {
apiMsg := &apiMessage{User: m.User, Message: m}
if m.Sender.IsValid() {
apiMsg.Sender = m.Sender.String()
}
return json.Marshal(apiMsg)
}
// assembleMessageBody builds an NDJSON fan-out request body from pre-marshaled apiMessage
// lines, avoiding a second JSON marshal of the messages.
func assembleMessageBody(lines [][]byte) []byte {
return append(bytes.Join(lines, []byte("\n")), '\n')
}
// decodeMessageBody reads NDJSON apiMessage lines from r, reattaches the non-JSON fields
// (Sender, User) onto each message, and hands them to deliver. Malformed or message-less lines
// are skipped and logged, not fatal: fan-out is fire-and-forget, so the valid remainder of a
// request is still delivered. It returns an error only for stream-level failures (e.g. a line
// exceeding maxLineBytes).
func decodeMessageBody(r io.Reader, maxLineBytes int, deliver DeliverFunc) error {
scanner := bufio.NewScanner(r)
// No initial buffer, so the caller's limit is the only number that matters here: bufio's
// token limit is the LARGER of the max and the initial buffer, so passing a buffer bigger
// than the limit quietly raises it (a 64KB buffer accepted 64KB lines under a 17KB limit).
// It grows on demand instead. One more byte because bufio's max is exclusive, and a line of
// exactly maxLineBytes is what the name promises to accept.
scanner.Buffer(nil, maxLineBytes+1)
for scanner.Scan() {
line := bytes.TrimSpace(scanner.Bytes())
if len(line) == 0 {
continue
}
var apiMsg apiMessage
if err := json.Unmarshal(line, &apiMsg); err != nil || apiMsg.Message == nil {
log.Tag(tag).Warn("Skipping malformed fan-out line")
continue
}
apiMsg.Message.User = apiMsg.User
if apiMsg.Sender != "" {
if addr, err := netip.ParseAddr(apiMsg.Sender); err == nil {
apiMsg.Message.Sender = addr
}
}
deliver(apiMsg.Message)
}
return scanner.Err()
}
cmd/cluster.go
+296
View File
@@ -0,0 +1,296 @@
//go:build !noserver
package cmd
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"sort"
"sync"
"text/tabwriter"
"time"
"github.com/urfave/cli/v2"
"github.com/urfave/cli/v2/altsrc"
"heckel.io/ntfy/v2/cluster"
"heckel.io/ntfy/v2/server"
)
const (
clusterStatusTimeout = 10 * time.Second
clusterStatusPeerTimeout = 5 * time.Second
clusterPeerLeftRecently = 10 * time.Minute // A peer gone longer than this was likely removed on purpose
)
func init() {
commands = append(commands, cmdCluster)
}
var flagsCluster = append(
append([]cli.Flag{}, flagsDefault...),
&cli.StringFlag{Name: "config", Aliases: []string{"c"}, EnvVars: []string{"NTFY_CONFIG_FILE"}, Value: server.DefaultConfigFile, DefaultText: server.DefaultConfigFile, Usage: "config file"},
altsrc.NewStringFlag(&cli.StringFlag{Name: "experimental-cluster-listen", Aliases: []string{"experimental_cluster_listen"}, EnvVars: []string{"NTFY_EXPERIMENTAL_CLUSTER_LISTEN"}, Usage: "ip:port of this node's cluster listener"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "experimental-cluster-secret", Aliases: []string{"experimental_cluster_secret"}, EnvVars: []string{"NTFY_EXPERIMENTAL_CLUSTER_SECRET"}, Usage: "shared secret authenticating cluster requests"}),
)
var cmdCluster = &cli.Command{
Name: "cluster",
Usage: "Show the state of an experimental cluster node",
UsageText: "ntfy cluster status [--all] [--json]",
Flags: flagsCluster,
Before: initConfigFileInputSourceFunc("config", flagsCluster, initLogFunc),
Category: categoryServer,
Subcommands: []*cli.Command{
{
Name: "status",
Usage: "Shows this node's view of the cluster",
UsageText: "ntfy cluster status [--all] [--json]",
Action: execClusterStatus,
Flags: []cli.Flag{
&cli.BoolFlag{Name: "json", Usage: "print the status as JSON"},
&cli.BoolFlag{Name: "all", Usage: "also fetch every live peer's view, and show all links between the nodes"},
},
Description: `Show this node's view of the cluster, as served on its cluster listener: its role and
leadership, and per peer the message batches it sent there and received from there.
A link that only works one way (e.g. a firewall dropping the cluster port in one direction)
shows as failed batches on the sending node and nothing received on the other one; it does
not show in /v1/health. Run this on the node itself: it reads experimental-cluster-listen and
experimental-cluster-secret from the server config.
With --all, it also fetches every live peer's view (from its advertised cluster URL) and shows
each link between the nodes once, with what the sender sent and what the receiver received.
Examples:
ntfy cluster status # Show peers and link problems
ntfy cluster status --all # Show all nodes and every link between them
ntfy cluster status --json # Print the raw status, e.g. for scripts
`,
},
},
}
// clusterNodeStatus is one node's view in "status --all": its status, or why it could not be fetched
type clusterNodeStatus struct {
NodeID cluster.NodeID `json:"node_id"`
Status *cluster.Status `json:"status,omitempty"`
Error string `json:"error,omitempty"`
}
func execClusterStatus(c *cli.Context) error {
listen := c.String("experimental-cluster-listen")
if listen == "" {
return errors.New("clustering is not enabled on this node (experimental-cluster-listen is not set)")
}
baseURL, err := clusterStatusBaseURL(listen)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), clusterStatusTimeout)
defer cancel()
status, err := cluster.NewClient(c.String("experimental-cluster-secret"), "").Status(ctx, baseURL)
if err != nil {
return err
}
if c.Bool("all") {
nodes := fetchClusterNodeStatuses(status, c.String("experimental-cluster-secret"))
if c.Bool("json") {
encoder := json.NewEncoder(c.App.Writer)
encoder.SetIndent("", " ")
return encoder.Encode(nodes)
}
printClusterNodeStatuses(c.App.Writer, nodes)
return nil
}
if c.Bool("json") {
encoder := json.NewEncoder(c.App.Writer)
encoder.SetIndent("", " ")
return encoder.Encode(status)
}
printClusterStatus(c.App.Writer, status)
return nil
}
// clusterStatusBaseURL turns the listen address into a URL on this host: a listener bound to all
// interfaces is reached on localhost
func clusterStatusBaseURL(listen string) (string, error) {
host, port, err := net.SplitHostPort(listen)
if err != nil {
return "", fmt.Errorf("invalid experimental-cluster-listen %q: %w", listen, err)
}
switch host {
case "", "0.0.0.0":
host = "127.0.0.1"
case "::":
host = "::1"
}
return "http://" + net.JoinHostPort(host, port), nil
}
func printClusterStatus(w io.Writer, status *cluster.Status) {
leader, healthy := "not leader", "healthy"
if status.Leader {
leader = "leader"
}
if !status.Healthy {
healthy = "NOT healthy (registry heartbeat stale)"
}
fmt.Fprintf(w, "Node %s: %s, %s, %s\n\n", status.NodeID, status.Role, leader, healthy)
if len(status.Peers) == 0 {
fmt.Fprintln(w, "No peers.")
return
}
tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
fmt.Fprintln(tw, "PEER\tROLE\tLIVE\tHEARTBEAT\tSENT OK\tFAILED\tQUEUED\tDROPPED\tLAST SENT\tRECEIVED\tLAST RECEIVED")
for _, p := range status.Peers {
live, heartbeat := "no", "last seen "+clusterStatusAge(status.Time, p.LastSeen)
if p.Live {
live, heartbeat = "yes", fmt.Sprintf("%ds ago", p.HeartbeatAge)
}
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%d\t%d\t%d\t%d\t%s\t%d\t%s\n", p.NodeID, p.Role, live, heartbeat, p.Out.BatchesOK, p.Out.BatchesFailed,
p.Out.Queued, p.Out.Dropped, clusterStatusAge(status.Time, p.Out.LastOK), p.In.Batches, clusterStatusAge(status.Time, p.In.LastAt))
}
tw.Flush()
problems := clusterStatusProblems(status, "")
if len(problems) == 0 {
return
}
fmt.Fprintln(w, "\nProblems:")
for _, problem := range problems {
fmt.Fprintf(w, " %s\n", problem)
}
}
// clusterStatusProblems lists what an operator should look at: outbound batches that fail
// (one-way links show up here on the sending side), drops, and peers that left the registry.
// With from set, each line names the link ("from -> peer") instead of only the peer.
func clusterStatusProblems(status *cluster.Status, from cluster.NodeID) []string {
problems := make([]string, 0)
for _, p := range status.Peers {
label := string(p.NodeID)
if from != "" {
label = fmt.Sprintf("%s -> %s", from, p.NodeID)
}
lastError := fmt.Sprintf("last error %s: %s", clusterStatusAge(status.Time, p.Out.LastErrorAt), p.Out.LastError)
if p.Out.BatchesFailed > 0 && p.Out.BatchesOK == 0 {
problems = append(problems, fmt.Sprintf("%s: no batch accepted, %d failed; %s", label, p.Out.BatchesFailed, lastError))
} else if p.Out.BatchesFailed > 0 && p.Out.LastFailed {
problems = append(problems, fmt.Sprintf("%s: the last batch failed (%d of %d failed); %s", label, p.Out.BatchesFailed, p.Out.BatchesFailed+p.Out.BatchesOK, lastError))
}
if p.Out.Dropped > 0 {
problems = append(problems, fmt.Sprintf("%s: %d message(s) dropped because its queue was full", label, p.Out.Dropped))
}
if !p.Live && (p.LastSeen == 0 || status.Time-p.LastSeen <= int64(clusterPeerLeftRecently.Seconds())) {
problems = append(problems, fmt.Sprintf("%s: left the registry %s (counters are from before it left)", label, clusterStatusAge(status.Time, p.LastSeen)))
}
}
return problems
}
// clusterStatusAge renders a status timestamp relative to the time the status was taken
func clusterStatusAge(now, t int64) string {
if t == 0 {
return "never"
}
return (time.Duration(now-t) * time.Second).String() + " ago"
}
// fetchClusterNodeStatuses fetches every live peer's view next to this node's, in parallel. A peer
// that does not answer is reported in its entry rather than failing the command.
func fetchClusterNodeStatuses(local *cluster.Status, secret string) []*clusterNodeStatus {
nodes := []*clusterNodeStatus{{NodeID: local.NodeID, Status: local}}
client := cluster.NewClient(secret, "")
var wg sync.WaitGroup
var mu sync.Mutex
for _, p := range local.Peers {
if !p.Live || p.AdvertiseURL == "" {
continue
}
wg.Add(1)
go func(p *cluster.PeerStatus) {
defer wg.Done()
node := &clusterNodeStatus{NodeID: p.NodeID}
ctx, cancel := context.WithTimeout(context.Background(), clusterStatusPeerTimeout)
defer cancel()
if status, err := client.Status(ctx, p.AdvertiseURL); err != nil {
node.Error = err.Error()
} else {
node.Status = status
}
mu.Lock()
defer mu.Unlock()
nodes = append(nodes, node)
}(p)
}
wg.Wait()
sort.Slice(nodes, func(i, j int) bool {
return nodes[i].NodeID < nodes[j].NodeID
})
return nodes
}
// printClusterNodeStatuses prints the nodes, then every link once: what the sender sent, next to
// what the receiver received from it ("?" if the receiver's view is not available)
func printClusterNodeStatuses(w io.Writer, nodes []*clusterNodeStatus) {
byID := make(map[cluster.NodeID]*cluster.Status)
tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
fmt.Fprintln(tw, "NODE\tROLE\tLEADER\tHEALTHY")
for _, node := range nodes {
if node.Status == nil {
fmt.Fprintf(tw, "%s\t?\t?\t?\n", node.NodeID)
continue
}
byID[node.NodeID] = node.Status
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", node.NodeID, node.Status.Role, clusterStatusYesNo(node.Status.Leader), clusterStatusYesNo(node.Status.Healthy))
}
tw.Flush()
fmt.Fprintln(w)
tw = tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
fmt.Fprintln(tw, "LINK\tSENT OK\tFAILED\tLAST SENT\tRECEIVED\tLAST RECEIVED")
problems := make([]string, 0)
for _, node := range nodes {
if node.Status == nil {
problems = append(problems, fmt.Sprintf("%s: cannot fetch its status: %s", node.NodeID, node.Error))
continue
}
for _, p := range node.Status.Peers {
received, lastReceived := "?", "?"
if peerStatus, ok := byID[p.NodeID]; ok {
in := clusterStatusInboundFrom(peerStatus, node.NodeID)
received, lastReceived = fmt.Sprintf("%d", in.Batches), clusterStatusAge(peerStatus.Time, in.LastAt)
}
fmt.Fprintf(tw, "%s -> %s\t%d\t%d\t%s\t%s\t%s\n", node.NodeID, p.NodeID, p.Out.BatchesOK, p.Out.BatchesFailed,
clusterStatusAge(node.Status.Time, p.Out.LastOK), received, lastReceived)
}
problems = append(problems, clusterStatusProblems(node.Status, node.NodeID)...)
}
tw.Flush()
if len(problems) == 0 {
return
}
fmt.Fprintln(w, "\nProblems:")
for _, problem := range problems {
fmt.Fprintf(w, " %s\n", problem)
}
}
// clusterStatusInboundFrom returns what a node received from the given peer, or nothing if it never did
func clusterStatusInboundFrom(status *cluster.Status, from cluster.NodeID) cluster.PeerInbound {
for _, p := range status.Peers {
if p.NodeID == from {
return p.In
}
}
return cluster.PeerInbound{}
}
func clusterStatusYesNo(b bool) string {
if b {
return "yes"
}
return "no"
}
cmd/cluster_test.go
+193
View File
@@ -0,0 +1,193 @@
package cmd
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/cluster"
)
// testClusterStatus is a passive node that cannot reach its active peer, while the peer reaches it
var testClusterStatus = &cluster.Status{
NodeID: "nyc3-app-2",
Role: cluster.RolePassive,
Leader: false,
Healthy: true,
Time: 1792330000,
Peers: []*cluster.PeerStatus{
{
NodeID: "nyc3-app-1",
AdvertiseURL: "http://10.108.0.2:2587",
Role: cluster.RoleActive,
Live: true,
HeartbeatAge: 2,
Out: cluster.PeerOutbound{BatchesFailed: 219, LastError: `Post "http://10.108.0.2:2587/v1/cluster/message": context deadline exceeded`, LastErrorAt: 1792329990},
In: cluster.PeerInbound{Batches: 5120, Messages: 828000, LastAt: 1792329999},
},
},
}
func newTestClusterStatusServer(t *testing.T) string {
return newTestClusterStatusServerFor(t, func() *cluster.Status {
return testClusterStatus
})
}
func newTestClusterStatusServerFor(t *testing.T, status func() *cluster.Status) string {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != cluster.StatusPath || r.Header.Get("X-Cluster-Secret") != "s3cret" {
w.WriteHeader(http.StatusUnauthorized)
return
}
require.Nil(t, json.NewEncoder(w).Encode(status()))
}))
t.Cleanup(srv.Close)
return strings.TrimPrefix(srv.URL, "http://")
}
// newTestClusterPair serves two nodes' views of a one-way link: app-2 cannot reach app-1, app-1 reaches app-2
func newTestClusterPair(t *testing.T) (app2Addr string) {
var app1Addr string
app2Addr = newTestClusterStatusServerFor(t, func() *cluster.Status {
return &cluster.Status{NodeID: "nyc3-app-2", Role: cluster.RolePassive, Healthy: true, Time: 1792330000, Peers: []*cluster.PeerStatus{{
NodeID: "nyc3-app-1", AdvertiseURL: "http://" + app1Addr, Role: cluster.RoleActive, Live: true, HeartbeatAge: 2,
Out: cluster.PeerOutbound{BatchesFailed: 219, LastError: "context deadline exceeded", LastErrorAt: 1792329990},
In: cluster.PeerInbound{Batches: 5120, Messages: 828000, LastAt: 1792329999},
}}}
})
app1Addr = newTestClusterStatusServerFor(t, func() *cluster.Status {
return &cluster.Status{NodeID: "nyc3-app-1", Role: cluster.RoleActive, Leader: true, Healthy: true, Time: 1792330000, Peers: []*cluster.PeerStatus{{
NodeID: "nyc3-app-2", AdvertiseURL: "http://" + app2Addr, Role: cluster.RolePassive, Live: true, HeartbeatAge: 1,
Out: cluster.PeerOutbound{BatchesOK: 5120, Messages: 828000, LastOK: 1792329999},
In: cluster.PeerInbound{},
}}}
})
return app2Addr
}
func TestCLI_Cluster_Status(t *testing.T) {
addr := newTestClusterStatusServer(t)
app, _, stdout, _ := newTestApp()
require.Nil(t, app.Run([]string{"ntfy", "cluster", "--experimental-cluster-listen", addr, "--experimental-cluster-secret", "s3cret", "status"}))
out := stdout.String()
require.Contains(t, out, "nyc3-app-2")
require.Contains(t, out, "passive")
require.Contains(t, out, "nyc3-app-1")
require.Contains(t, out, "219") // Failed batches
require.Contains(t, out, "context deadline exceeded")
require.Contains(t, out, "5120") // Batches received
require.Contains(t, out, "never") // No successful send to nyc3-app-1
require.Contains(t, out, "nyc3-app-1: no batch accepted") // The warning that matters
}
func TestCLI_Cluster_Status_JSON(t *testing.T) {
addr := newTestClusterStatusServer(t)
app, _, stdout, _ := newTestApp()
require.Nil(t, app.Run([]string{"ntfy", "cluster", "--experimental-cluster-listen", addr, "--experimental-cluster-secret", "s3cret", "status", "--json"}))
var status cluster.Status
require.Nil(t, json.Unmarshal(stdout.Bytes(), &status))
require.Equal(t, cluster.NodeID("nyc3-app-2"), status.NodeID)
require.Equal(t, int64(219), status.Peers[0].Out.BatchesFailed)
}
func TestCLI_Cluster_Status_WildcardListen(t *testing.T) {
// A listener bound to all interfaces is queried on localhost
addr := newTestClusterStatusServer(t)
port := addr[strings.LastIndex(addr, ":"):]
app, _, stdout, _ := newTestApp()
require.Nil(t, app.Run([]string{"ntfy", "cluster", "--experimental-cluster-listen", port, "--experimental-cluster-secret", "s3cret", "status"}))
require.Contains(t, stdout.String(), "nyc3-app-2")
}
func TestCLI_Cluster_Status_WrongSecret(t *testing.T) {
addr := newTestClusterStatusServer(t)
app, _, _, _ := newTestApp()
err := app.Run([]string{"ntfy", "cluster", "--experimental-cluster-listen", addr, "--experimental-cluster-secret", "wrong", "status"})
require.ErrorContains(t, err, "401")
}
func TestCLI_Cluster_Status_NotEnabled(t *testing.T) {
app, _, _, _ := newTestApp()
err := app.Run([]string{"ntfy", "cluster", "status"})
require.ErrorContains(t, err, "clustering is not enabled")
}
func TestCLI_Cluster_StatusAll(t *testing.T) {
// Every node's view side by side: the one-way link shows as failures on one side and nothing
// received on the other, from a single command on either node
addr := newTestClusterPair(t)
app, _, stdout, _ := newTestApp()
require.Nil(t, app.Run([]string{"ntfy", "cluster", "--experimental-cluster-listen", addr, "--experimental-cluster-secret", "s3cret", "status", "--all"}))
out := stdout.String()
require.Regexp(t, `nyc3-app-1\s+active\s+yes`, out) // Leader
require.Regexp(t, `nyc3-app-2\s+passive\s+no`, out)
require.Regexp(t, `nyc3-app-1 -> nyc3-app-2\s+5120\s+0\s+1s ago\s+5120`, out)
require.Regexp(t, `nyc3-app-2 -> nyc3-app-1\s+0\s+219\s+never\s+0\s+never`, out)
require.Contains(t, out, "nyc3-app-2 -> nyc3-app-1: no batch accepted, 219 failed")
}
func TestCLI_Cluster_StatusAll_PeerUnreachable(t *testing.T) {
// A peer whose status cannot be fetched is reported, the rest is still printed
addr := newTestClusterStatusServerFor(t, func() *cluster.Status {
return &cluster.Status{NodeID: "nyc3-app-1", Role: cluster.RoleActive, Leader: true, Healthy: true, Time: 1792330000, Peers: []*cluster.PeerStatus{{
NodeID: "nyc3-app-2", AdvertiseURL: "http://127.0.0.1:1", Role: cluster.RolePassive, Live: true,
Out: cluster.PeerOutbound{BatchesOK: 10, LastOK: 1792329999},
}}}
})
app, _, stdout, _ := newTestApp()
require.Nil(t, app.Run([]string{"ntfy", "cluster", "--experimental-cluster-listen", addr, "--experimental-cluster-secret", "s3cret", "status", "--all"}))
out := stdout.String()
require.Regexp(t, `nyc3-app-1 -> nyc3-app-2\s+10\s+0\s+1s ago\s+\?`, out)
require.Contains(t, out, "nyc3-app-2: cannot fetch its status")
}
func TestCLI_Cluster_StatusAll_JSON(t *testing.T) {
addr := newTestClusterPair(t)
app, _, stdout, _ := newTestApp()
require.Nil(t, app.Run([]string{"ntfy", "cluster", "--experimental-cluster-listen", addr, "--experimental-cluster-secret", "s3cret", "status", "--all", "--json"}))
var nodes []*clusterNodeStatus
require.Nil(t, json.Unmarshal(stdout.Bytes(), &nodes))
require.Len(t, nodes, 2)
require.Equal(t, cluster.NodeID("nyc3-app-1"), nodes[0].NodeID)
require.Equal(t, int64(5120), nodes[0].Status.Peers[0].Out.BatchesOK)
require.Equal(t, cluster.NodeID("nyc3-app-2"), nodes[1].NodeID)
require.Equal(t, int64(219), nodes[1].Status.Peers[0].Out.BatchesFailed)
}
func TestCLI_Cluster_Status_PeerLeft(t *testing.T) {
// A peer that left recently is a problem; one that left long ago (decommissioned) is only listed
status := func(leftAgo int64) func() *cluster.Status {
return func() *cluster.Status {
return &cluster.Status{NodeID: "nyc3-app-1", Role: cluster.RoleActive, Leader: true, Healthy: true, Time: 1792330000, Peers: []*cluster.PeerStatus{{
NodeID: "nyc3-app-2", Live: false, LastSeen: 1792330000 - leftAgo, Out: cluster.PeerOutbound{BatchesOK: 10, LastOK: 1792330000 - leftAgo},
}}}
}
}
app, _, stdout, _ := newTestApp()
require.Nil(t, app.Run([]string{"ntfy", "cluster", "--experimental-cluster-listen", newTestClusterStatusServerFor(t, status(30)), "--experimental-cluster-secret", "s3cret", "status"}))
require.Contains(t, stdout.String(), "last seen 30s ago")
require.Contains(t, stdout.String(), "nyc3-app-2: left the registry 30s ago")
app, _, stdout, _ = newTestApp()
require.Nil(t, app.Run([]string{"ntfy", "cluster", "--experimental-cluster-listen", newTestClusterStatusServerFor(t, status(3*3600)), "--experimental-cluster-secret", "s3cret", "status"}))
require.Contains(t, stdout.String(), "last seen 3h0m0s ago")
require.NotContains(t, stdout.String(), "left the registry")
}
func TestCLI_Cluster_StatusProblems_OutcomeOrderWithinOneSecond(t *testing.T) {
// Success and failure in the same Unix second: the recorded last outcome decides, not the timestamps
failedLast := &cluster.Status{Time: 100, Peers: []*cluster.PeerStatus{{
NodeID: "peer", Live: true,
Out: cluster.PeerOutbound{BatchesOK: 1, BatchesFailed: 1, LastOK: 100, LastErrorAt: 100, LastError: "connection refused", LastFailed: true},
}}}
require.NotEmpty(t, clusterStatusProblems(failedLast, ""))
recoveredLast := &cluster.Status{Time: 100, Peers: []*cluster.PeerStatus{{
NodeID: "peer", Live: true,
Out: cluster.PeerOutbound{BatchesOK: 1, BatchesFailed: 1, LastOK: 100, LastErrorAt: 100, LastError: "connection refused", LastFailed: false},
}}}
require.Empty(t, clusterStatusProblems(recoveredLast, ""))
}
cmd/serve.go
+116 -28
View File
@@ -19,6 +19,7 @@ import (
"github.com/urfave/cli/v2"
"github.com/urfave/cli/v2/altsrc"
"heckel.io/ntfy/v2/ban"
"heckel.io/ntfy/v2/cluster"
"heckel.io/ntfy/v2/log"
"heckel.io/ntfy/v2/payments"
"heckel.io/ntfy/v2/server"
@@ -34,7 +35,7 @@ var flagsServe = append(
append([]cli.Flag{}, flagsDefault...),
&cli.StringFlag{Name: "config", Aliases: []string{"c"}, EnvVars: []string{"NTFY_CONFIG_FILE"}, Value: server.DefaultConfigFile, Usage: "config file"},
altsrc.NewStringFlag(&cli.StringFlag{Name: "base-url", Aliases: []string{"base_url", "B"}, EnvVars: []string{"NTFY_BASE_URL"}, Usage: "externally visible base URL for this host (e.g. https://ntfy.sh)"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-http", Aliases: []string{"listen_http", "l"}, EnvVars: []string{"NTFY_LISTEN_HTTP"}, Value: server.DefaultListenHTTP, Usage: "ip:port used as HTTP listen address"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-http", Aliases: []string{"listen_http", "l"}, EnvVars: []string{"NTFY_LISTEN_HTTP"}, Value: server.DefaultListenHTTP, Usage: "ip:port used as HTTP listen address; several comma-separated addresses are allowed"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-https", Aliases: []string{"listen_https", "L"}, EnvVars: []string{"NTFY_LISTEN_HTTPS"}, Usage: "ip:port used as HTTPS listen address"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-unix", Aliases: []string{"listen_unix", "U"}, EnvVars: []string{"NTFY_LISTEN_UNIX"}, Usage: "listen on unix socket path"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "listen-unix-mode", Aliases: []string{"listen_unix_mode"}, EnvVars: []string{"NTFY_LISTEN_UNIX_MODE"}, DefaultText: "system default", Usage: "file permissions of unix socket, e.g. 0700"}),
@@ -43,6 +44,14 @@ var flagsServe = append(
altsrc.NewStringFlag(&cli.StringFlag{Name: "firebase-key-file", Aliases: []string{"firebase_key_file", "F"}, EnvVars: []string{"NTFY_FIREBASE_KEY_FILE"}, Usage: "Firebase credentials file; if set additionally publish to FCM topic"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "database-url", Aliases: []string{"database_url"}, EnvVars: []string{"NTFY_DATABASE_URL"}, Usage: "PostgreSQL connection string for database-backed stores (e.g. postgres://user:pass@host:5432/ntfy)"}),
altsrc.NewStringSliceFlag(&cli.StringSliceFlag{Name: "database-replica-urls", Aliases: []string{"database_replica_urls"}, EnvVars: []string{"NTFY_DATABASE_REPLICA_URLS"}, Usage: "PostgreSQL read replica connection strings for offloading read queries"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "experimental-cluster-node-id", Aliases: []string{"experimental_cluster_node_id"}, EnvVars: []string{"NTFY_EXPERIMENTAL_CLUSTER_NODE_ID"}, Usage: "stable per-node identifier for the cluster node registry (required in cluster mode)"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "experimental-cluster-listen", Aliases: []string{"experimental_cluster_listen"}, EnvVars: []string{"NTFY_EXPERIMENTAL_CLUSTER_LISTEN"}, Usage: "ip:port for the dedicated cluster fan-out listener; bind it to the private network (e.g. 10.0.0.5:2587)"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "experimental-cluster-advertise-url", Aliases: []string{"experimental_cluster_advertise_url"}, EnvVars: []string{"NTFY_EXPERIMENTAL_CLUSTER_ADVERTISE_URL"}, Usage: "base URL peer nodes use to reach this node's fan-out listener (defaults to http://<experimental-cluster-listen>)"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "experimental-cluster-secret", Aliases: []string{"experimental_cluster_secret"}, EnvVars: []string{"NTFY_EXPERIMENTAL_CLUSTER_SECRET"}, Usage: "shared secret authenticating node-to-node fan-out requests"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "experimental-cluster-role", Aliases: []string{"experimental_cluster_role"}, EnvVars: []string{"NTFY_EXPERIMENTAL_CLUSTER_ROLE"}, Value: string(server.DefaultClusterRole), Usage: "role of this cluster node: active (takes client traffic, may lead) or passive (hidden: never leads, never fences)"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "experimental-cluster-batch-linger", Aliases: []string{"experimental_cluster_batch_linger"}, EnvVars: []string{"NTFY_EXPERIMENTAL_CLUSTER_BATCH_LINGER"}, Value: util.FormatDuration(server.DefaultClusterBatchLinger), Usage: "how long fan-out messages wait to form a batch per peer node (0 = send immediately)"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "experimental-cluster-drain-delay", Aliases: []string{"experimental_cluster_drain_delay"}, EnvVars: []string{"NTFY_EXPERIMENTAL_CLUSTER_DRAIN_DELAY"}, Value: util.FormatDuration(server.DefaultClusterDrainDelay), Usage: "on shutdown (SIGTERM), how long an active cluster node withdraws (not ready, but still serves everything), so the load balancers stop sending it requests before it turns read-only"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "experimental-cluster-drain-duration", Aliases: []string{"experimental_cluster_drain_duration"}, EnvVars: []string{"NTFY_EXPERIMENTAL_CLUSTER_DRAIN_DURATION"}, Value: util.FormatDuration(server.DefaultClusterDrainDuration), Usage: "on shutdown (SIGTERM), how long an active cluster node takes to close its subscriber connections, so they reconnect to the other nodes gradually (0 = all at once)"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "cache-file", Aliases: []string{"cache_file", "C"}, EnvVars: []string{"NTFY_CACHE_FILE"}, Usage: "cache file used for message caching"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "cache-duration", Aliases: []string{"cache_duration", "b"}, EnvVars: []string{"NTFY_CACHE_DURATION"}, Value: util.FormatDuration(server.DefaultCacheDuration), Usage: "buffer messages for this time to allow `since` requests"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "cache-batch-size", Aliases: []string{"cache_batch_size"}, EnvVars: []string{"NTFY_BATCH_SIZE"}, Usage: "max size of messages to batch together when writing to message cache (if zero, writes are synchronous)"}),
@@ -148,7 +157,7 @@ func execServe(c *cli.Context) error {
// Read all the options
config := c.String("config")
baseURL := strings.TrimSuffix(c.String("base-url"), "/")
listenHTTP := c.String("listen-http")
listenHTTPRaw := c.String("listen-http")
listenHTTPS := c.String("listen-https")
listenUnix := c.String("listen-unix")
listenUnixMode := c.Int("listen-unix-mode")
@@ -157,6 +166,14 @@ func execServe(c *cli.Context) error {
firebaseKeyFile := c.String("firebase-key-file")
databaseURL := c.String("database-url")
databaseReplicaURLs := c.StringSlice("database-replica-urls")
clusterNodeID := c.String("experimental-cluster-node-id")
clusterListen := c.String("experimental-cluster-listen")
clusterAdvertiseURL := c.String("experimental-cluster-advertise-url")
clusterSecret := c.String("experimental-cluster-secret")
clusterBatchLingerStr := c.String("experimental-cluster-batch-linger")
clusterRoleStr := c.String("experimental-cluster-role")
clusterDrainDelayStr := c.String("experimental-cluster-drain-delay")
clusterDrainDurationStr := c.String("experimental-cluster-drain-duration")
webPushPrivateKey := c.String("web-push-private-key")
webPushPublicKey := c.String("web-push-public-key")
webPushFile := c.String("web-push-file")
@@ -252,6 +269,22 @@ func execServe(c *cli.Context) error {
if err != nil {
return fmt.Errorf("invalid keepalive interval: %s", keepaliveIntervalStr)
}
clusterBatchLinger, err := util.ParseDuration(clusterBatchLingerStr)
if err != nil || clusterBatchLinger < 0 {
return fmt.Errorf("invalid cluster batch linger: %s", clusterBatchLingerStr)
}
clusterDrainDelay, err := util.ParseDuration(clusterDrainDelayStr)
if err != nil || clusterDrainDelay < 0 {
return fmt.Errorf("invalid cluster drain delay: %s", clusterDrainDelayStr)
}
clusterDrainDuration, err := util.ParseDuration(clusterDrainDurationStr)
if err != nil || clusterDrainDuration < 0 {
return fmt.Errorf("invalid cluster drain duration: %s", clusterDrainDurationStr)
}
clusterRole := cluster.Role(clusterRoleStr)
if !clusterRole.Valid() {
return fmt.Errorf("invalid experimental-cluster-role: %s (must be %s or %s)", clusterRoleStr, cluster.RoleActive, cluster.RolePassive)
}
managerInterval, err := util.ParseDuration(managerIntervalStr)
if err != nil {
return fmt.Errorf("invalid manager interval: %s", managerIntervalStr)
@@ -322,6 +355,16 @@ func execServe(c *cli.Context) error {
return errors.New("if database-url is set, auth-file, cache-file, and web-push-file must not be set")
} else if len(databaseReplicaURLs) > 0 && databaseURL == "" {
return errors.New("database-replica-urls can only be used if database-url is also set")
} else if clusterListen != "" && databaseURL == "" {
return errors.New("experimental-cluster-listen requires database-url to be set")
} else if clusterListen != "" && clusterSecret == "" {
return errors.New("experimental-cluster-listen requires experimental-cluster-secret to be set")
} else if clusterListen != "" && clusterNodeID == "" {
return errors.New("experimental-cluster-listen requires experimental-cluster-node-id to be set")
} else if clusterListen == "" && clusterSecret != "" {
return errors.New("experimental-cluster-secret can only be used if experimental-cluster-listen is set")
} else if clusterListen != "" && clusterAdvertiseURL == "" && wildcardAddr(clusterListen) {
return errors.New("experimental-cluster-advertise-url must be set if experimental-cluster-listen binds a wildcard address")
} else if firebaseKeyFile != "" && !util.FileExists(firebaseKeyFile) {
return errors.New("if set, FCM key file must exist")
} else if firebaseKeyFile != "" && !server.FirebaseAvailable {
@@ -348,8 +391,15 @@ func execServe(c *cli.Context) error {
return errors.New("if smtp-server-listen is set, smtp-server-domain must also be set")
} else if attachmentCacheDir != "" && baseURL == "" {
return errors.New("if attachment-cache-dir is set, base-url must also be set")
} else if err := validateBaseURL(baseURL); err != nil {
return err
} else if baseURL != "" {
u, err := url.Parse(baseURL)
if err != nil {
return fmt.Errorf("if set, base-url must be a valid URL, e.g. https://ntfy.mydomain.com: %v", err)
} else if u.Scheme != "http" && u.Scheme != "https" {
return errors.New("if set, base-url must be a valid URL starting with http:// or https://, e.g. https://ntfy.mydomain.com")
} else if u.Path != "" {
return fmt.Errorf("if set, base-url must not have a path (%s), as hosting ntfy on a sub-path is not supported, e.g. https://ntfy.mydomain.com", u.Path)
}
} else if upstreamBaseURL != "" && !strings.HasPrefix(upstreamBaseURL, "http://") && !strings.HasPrefix(upstreamBaseURL, "https://") {
return errors.New("if set, upstream-base-url must start with http:// or https://")
} else if upstreamBaseURL != "" && strings.HasSuffix(upstreamBaseURL, "/") {
@@ -370,8 +420,11 @@ func execServe(c *cli.Context) error {
return errors.New("if stripe-secret-key is set, stripe-webhook-key and base-url must also be set")
} else if twilioAccount != "" && (twilioAuthToken == "" || twilioPhoneNumber == "" || twilioVerifyService == "" || baseURL == "" || (authFile == "" && databaseURL == "")) {
return errors.New("if twilio-account is set, twilio-auth-token, twilio-phone-number, twilio-verify-service, base-url, and auth-file (or database-url) must also be set")
} else if messageSizeLimit > 5*1024*1024 {
return errors.New("message-size-limit cannot be higher than 5M")
} else if messageSizeLimit > server.DefaultMessageSizeLimit {
log.Warn("message-size-limit is greater than 4K, this is not recommended and largely untested, and may lead to issues with some clients")
if messageSizeLimit > 5*1024*1024 {
return errors.New("message-size-limit cannot be higher than 5M")
}
} else if !server.WebPushAvailable && (webPushPrivateKey != "" || webPushPublicKey != "" || webPushFile != "") {
return errors.New("cannot enable WebPush, support is not available in this build (nowebpush)")
} else if webPushExpiryWarningDuration > 0 && webPushExpiryWarningDuration > webPushExpiryDuration {
@@ -393,8 +446,8 @@ func execServe(c *cli.Context) error {
} else if runtime.GOOS == "windows" && listenUnix != "" {
return errors.New("listen-unix is not supported on Windows")
}
if messageSizeLimit > server.DefaultMessageSizeLimit {
log.Warn("message-size-limit is greater than 4K, this is not recommended and largely untested, and may lead to issues with some clients")
if cacheDuration > 0 && attachmentExpiryDuration > cacheDuration {
log.Warn("attachment-expiry-duration (%s) is longer than cache-duration (%s); attachments never outlive their message, so they will be deleted after %s", attachmentExpiryDurationStr, cacheDurationStr, cacheDurationStr)
}
// Backwards compatibility
@@ -426,9 +479,9 @@ func execServe(c *cli.Context) error {
return err
}
// Special case: Unset default
if listenHTTP == "-" {
listenHTTP = ""
listenHTTP, err := parseListenHTTP(listenHTTPRaw)
if err != nil {
return err
}
// Resolve hosts
@@ -552,6 +605,14 @@ func execServe(c *cli.Context) error {
conf.ProfileListenHTTP = profileListenHTTP
conf.DatabaseURL = databaseURL
conf.DatabaseReplicaURLs = databaseReplicaURLs
conf.ClusterNodeID = clusterNodeID
conf.ClusterListen = clusterListen
conf.ClusterAdvertiseURL = clusterAdvertiseURL
conf.ClusterSecret = clusterSecret
conf.ClusterBatchLinger = clusterBatchLinger
conf.ClusterRole = clusterRole
conf.ClusterDrainDelay = clusterDrainDelay
conf.ClusterDrainDuration = clusterDrainDuration
conf.WebPushPrivateKey = webPushPrivateKey
conf.WebPushPublicKey = webPushPublicKey
conf.WebPushFile = webPushFile
@@ -578,28 +639,24 @@ func execServe(c *cli.Context) error {
s, err := server.New(conf)
if err != nil {
log.Fatal("%s", err.Error())
} else if err := s.Run(); err != nil {
log.Fatal("%s", err.Error())
}
stopping := make(chan struct{})
go sigHandlerShutdown(s, stopping)
err = s.Run()
select {
case <-stopping:
// Run returns as soon as the listeners close; Stop blocks until the signal-triggered
// shutdown (incl. flushing the message cache) has completed
s.Stop()
default:
if err != nil {
log.Fatal("%s", err.Error())
}
}
log.Info("Exiting.")
return nil
}
func validateBaseURL(baseURL string) error {
if baseURL == "" {
return nil
}
u, err := url.Parse(baseURL)
if err != nil {
return fmt.Errorf("if set, base-url must be a valid URL, e.g. https://ntfy.mydomain.com: %v", err)
} else if u.Scheme != "http" && u.Scheme != "https" {
return errors.New("if set, base-url must be a valid URL starting with http:// or https://, e.g. https://ntfy.mydomain.com")
} else if u.Path != "" {
return fmt.Errorf("if set, base-url must not have a path (%s), as hosting ntfy on a sub-path is not supported, e.g. https://ntfy.mydomain.com", u.Path)
}
return nil
}
func parseIPHostPrefix(host string) (prefixes []netip.Prefix, err error) {
// Try parsing as prefix, e.g. 10.0.1.0/24 or 2001:db8::/32
prefix, err := netip.ParsePrefix(host)
@@ -745,3 +802,34 @@ func maybeFromMetadata(m map[string]any, key string) string {
}
return s
}
// wildcardAddr reports whether the given listen address binds all interfaces (e.g. ":2587",
// "0.0.0.0:2587", "[::]:2587"), in which case peers cannot derive a reachable URL from it.
func wildcardAddr(addr string) bool {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return true // Unparseable -> cannot derive a URL either
}
return host == "" || host == "0.0.0.0" || host == "::"
}
// parseListenHTTP parses the comma-separated listen-http addresses ("-" unsets the default)
func parseListenHTTP(raw string) ([]string, error) {
addrs := make([]string, 0)
if raw == "" || raw == "-" {
return addrs, nil
}
seen := make(map[string]struct{})
for _, addr := range strings.Split(raw, ",") {
addr = strings.TrimSpace(addr)
if addr == "" {
return nil, fmt.Errorf("invalid listen-http: %s, empty address", raw)
}
if _, ok := seen[addr]; ok {
return nil, fmt.Errorf("invalid listen-http: %s, duplicate address %s", raw, addr)
}
seen[addr] = struct{}{}
addrs = append(addrs, addr)
}
return addrs, nil
}
cmd/serve_test.go
+150 -46
View File
@@ -1,6 +1,7 @@
package cmd
import (
"bytes"
"fmt"
"math/rand"
"os"
@@ -13,6 +14,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/client"
"heckel.io/ntfy/v2/log"
"heckel.io/ntfy/v2/test"
"heckel.io/ntfy/v2/user"
"heckel.io/ntfy/v2/util"
@@ -520,6 +522,79 @@ func TestCLI_Serve_WebSocket(t *testing.T) {
require.Equal(t, "mytopic", m.Topic)
}
func TestParseListenHTTP_Success(t *testing.T) {
tests := []struct {
name string
input string
expected []string
}{
{
name: "single address",
input: ":80",
expected: []string{":80"},
},
{
name: "several addresses",
input: "10.0.0.5:2586,10.0.0.5:2588,10.0.0.5:2589",
expected: []string{"10.0.0.5:2586", "10.0.0.5:2588", "10.0.0.5:2589"},
},
{
name: "whitespace around commas",
input: " 10.0.0.5:2586 , 10.0.0.5:2588 ",
expected: []string{"10.0.0.5:2586", "10.0.0.5:2588"},
},
{
name: "unset default",
input: "-",
expected: []string{},
},
{
name: "empty",
input: "",
expected: []string{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := parseListenHTTP(tt.input)
require.Nil(t, err)
assert.Equal(t, tt.expected, result)
})
}
}
func TestParseListenHTTP_Errors(t *testing.T) {
tests := []struct {
name string
input string
error string
}{
{
name: "empty entry",
input: "10.0.0.5:2586,,10.0.0.5:2588",
error: "invalid listen-http: 10.0.0.5:2586,,10.0.0.5:2588, empty address",
},
{
name: "trailing comma",
input: "10.0.0.5:2586,",
error: "invalid listen-http: 10.0.0.5:2586,, empty address",
},
{
name: "duplicate address",
input: "10.0.0.5:2586,10.0.0.5:2586",
error: "invalid listen-http: 10.0.0.5:2586,10.0.0.5:2586, duplicate address 10.0.0.5:2586",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := parseListenHTTP(tt.input)
require.Error(t, err)
require.Nil(t, result)
assert.Equal(t, tt.error, err.Error())
})
}
}
func TestIP_Host_Parsing(t *testing.T) {
cases := map[string]string{
"1.1.1.1": "1.1.1.1/32",
@@ -536,55 +611,84 @@ func TestIP_Host_Parsing(t *testing.T) {
}
}
func TestCLI_Serve_ValidationNotSkippedWhenBaseURLSet(t *testing.T) {
authFile := filepath.Join(t.TempDir(), "user.db")
cases := map[string]struct {
args []string
expectedErr string
}{
"identical upstream-base-url": {
args: []string{"--base-url=https://ntfy.example.com", "--upstream-base-url=https://ntfy.example.com"},
expectedErr: "base-url and upstream-base-url cannot be identical",
},
"upstream-base-url with trailing slash": {
args: []string{"--base-url=https://ntfy.example.com", "--upstream-base-url=https://ntfy.sh/"},
expectedErr: "upstream-base-url must not end with a slash",
},
"require-login without enable-login": {
args: []string{"--base-url=https://ntfy.example.com", "--auth-file=" + authFile, "--require-login"},
expectedErr: "cannot set require-login without also setting enable-login",
},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
require.ErrorContains(t, runServeUntilError(t, c.args...), c.expectedErr)
})
}
}
func TestCLI_Serve_ValidationNotSkippedWhenMessageSizeLimitAboveDefault(t *testing.T) {
err := runServeUntilError(t, "--message-size-limit=8k", "--visitor-prefix-bits-ipv4=0")
require.ErrorContains(t, err, "visitor-prefix-bits-ipv4 must be between 1 and 32")
}
// runServeUntilError runs "ntfy serve" and returns its error. If the config passes validation,
// the server starts and blocks, so the test fails after a timeout instead of hanging.
func runServeUntilError(t *testing.T, args ...string) error {
func TestCLI_Serve_ClusterValidation(t *testing.T) {
configFile := newEmptyFile(t) // Avoid issues with existing server.yml file on system
port := 10000 + rand.Intn(20000)
errChan := make(chan error, 1)
go func() {
// Setting experimental-cluster-listen implicitly enables clustering, which requires
// database-url; all validation must fail before any database connection is attempted
app, _, _, _ := newTestApp()
err := app.Run([]string{"ntfy", "serve", "--config=" + configFile, "--experimental-cluster-listen=127.0.0.1:2587"})
require.Error(t, err)
require.Contains(t, err.Error(), "database-url")
// It also requires a shared secret
app, _, _, _ = newTestApp()
err = app.Run([]string{"ntfy", "serve", "--config=" + configFile, "--experimental-cluster-listen=127.0.0.1:2587", "--database-url=postgres://user:pass@localhost:1/na"})
require.Error(t, err)
require.Contains(t, err.Error(), "experimental-cluster-secret")
// ... and an explicit stable node ID
app, _, _, _ = newTestApp()
err = app.Run([]string{"ntfy", "serve", "--config=" + configFile, "--experimental-cluster-listen=127.0.0.1:2587", "--database-url=postgres://user:pass@localhost:1/na", "--experimental-cluster-secret=s3cret"})
require.Error(t, err)
require.Contains(t, err.Error(), "experimental-cluster-node-id")
// A secret without a listen address is a config error: clustering would silently be off
app, _, _, _ = newTestApp()
err = app.Run([]string{"ntfy", "serve", "--config=" + configFile, "--experimental-cluster-secret=s3cret"})
require.Error(t, err)
require.Contains(t, err.Error(), "experimental-cluster-listen")
// A wildcard bind cannot derive an advertise URL
app, _, _, _ = newTestApp()
err = app.Run([]string{"ntfy", "serve", "--config=" + configFile, "--experimental-cluster-listen=:2587", "--database-url=postgres://user:pass@localhost:1/na", "--experimental-cluster-secret=s3cret", "--experimental-cluster-node-id=node-a"})
require.Error(t, err)
require.Contains(t, err.Error(), "experimental-cluster-advertise-url")
// The linger must not be negative
app, _, _, _ = newTestApp()
err = app.Run([]string{"ntfy", "serve", "--config=" + configFile, "--experimental-cluster-batch-linger=-1s"})
require.Error(t, err)
require.Contains(t, err.Error(), "cluster batch linger")
// The drain duration must not be negative either (0 disables draining)
app, _, _, _ = newTestApp()
err = app.Run([]string{"ntfy", "serve", "--config=" + configFile, "--experimental-cluster-drain-duration=-1s"})
require.Error(t, err)
require.Contains(t, err.Error(), "cluster drain duration")
// The drain delay must not be negative either
app, _, _, _ = newTestApp()
err = app.Run([]string{"ntfy", "serve", "--config=" + configFile, "--experimental-cluster-drain-delay=-1s"})
require.Error(t, err)
require.Contains(t, err.Error(), "cluster drain delay")
// 0 is valid (close all at once): the error is about the role, which is checked after the drain duration
app, _, _, _ = newTestApp()
err = app.Run([]string{"ntfy", "serve", "--config=" + configFile, "--experimental-cluster-drain-duration=0", "--experimental-cluster-listen=:2587", "--database-url=postgres://user:pass@localhost:1/na", "--experimental-cluster-secret=s3cret", "--experimental-cluster-node-id=node-a", "--experimental-cluster-role=leader"})
require.Error(t, err)
require.Contains(t, err.Error(), "experimental-cluster-role")
// The role must be one of the known ones: a typo would make a node half passive
app, _, _, _ = newTestApp()
err = app.Run([]string{"ntfy", "serve", "--config=" + configFile, "--experimental-cluster-listen=:2587", "--database-url=postgres://user:pass@localhost:1/na", "--experimental-cluster-secret=s3cret", "--experimental-cluster-node-id=node-a", "--experimental-cluster-role=leader"})
require.Error(t, err)
require.Contains(t, err.Error(), "experimental-cluster-role")
}
func TestCLI_Serve_AttachmentExpiryLongerThanCacheDurationWarning(t *testing.T) {
configFile := newEmptyFile(t) // Avoid issues with existing server.yml file on system
var buf bytes.Buffer
log.SetOutput(&buf)
defer log.SetOutput(os.Stderr)
// An invalid auth-default-access fails right after validation, before the server starts
runServe := func(args ...string) {
app, _, _, _ := newTestApp()
serveArgs := []string{"ntfy", "serve", "--config=" + configFile, fmt.Sprintf("--listen-http=:%d", port)}
errChan <- app.Run(append(serveArgs, args...))
}()
select {
case err := <-errChan:
return err
case <-time.After(3 * time.Second):
t.Fatal("server started, but config should have been rejected")
return nil
err := app.Run(append([]string{"ntfy", "serve", "--config=" + configFile, "--auth-default-access=invalid"}, args...))
require.Error(t, err)
require.Contains(t, err.Error(), "auth-default-access")
}
// Attachments outliving the message cache are cut short, so warn
runServe("--cache-duration=12h", "--attachment-expiry-duration=24h")
require.Contains(t, buf.String(), "attachment-expiry-duration (24h) is longer than cache-duration (12h)")
// No warning if the attachments expire first, or if messages are kept forever
buf.Reset()
runServe("--cache-duration=12h", "--attachment-expiry-duration=3h")
runServe("--cache-duration=0", "--attachment-expiry-duration=24h")
require.NotContains(t, buf.String(), "attachment-expiry-duration")
}
func newEmptyFile(t *testing.T) string {
cmd/serve_unix.go
+26
View File
@@ -3,6 +3,7 @@
package cmd
import (
"context"
"os"
"os/signal"
"syscall"
@@ -28,6 +29,31 @@ func sigHandlerConfigReload(config string) {
}
}
// sigHandlerShutdown stops the server gracefully on SIGTERM/SIGINT (systemd stop, Ctrl-C), so
// pending writes such as the message cache batch are persisted. It closes stopping first, so
// the caller can tell the resulting Run error apart from a real failure.
//
// A cluster node first drains (see Server.Drain); a second signal skips the rest of the drain.
func sigHandlerShutdown(s *server.Server, stopping chan<- struct{}) {
sigs := make(chan os.Signal, 2)
signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT)
sig := <-sigs
log.Info("Received %s, shutting down ...", sig)
close(stopping)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
select {
case sig := <-sigs:
log.Info("Received %s again, shutting down without draining", sig)
cancel()
case <-ctx.Done():
}
}()
s.Drain(ctx)
s.Stop()
}
func reloadLogLevel(inputSource altsrc.InputSourceContext) error {
newLevelStr, err := inputSource.String("log-level")
if err != nil {
cmd/serve_unix_test.go
+57
View File
@@ -0,0 +1,57 @@
//go:build (darwin || linux || dragonfly || freebsd || netbsd || openbsd) && !noserver
package cmd
import (
"fmt"
"math/rand"
"net/http"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/message"
"heckel.io/ntfy/v2/model"
)
func TestCLI_Serve_SIGTERMFlushesBatchedMessages(t *testing.T) {
// systemd stops ntfy with SIGTERM on every deploy: serve must shut down gracefully and
// persist messages still waiting in the cache write batch instead of dying mid-batch
port := 10000 + rand.Intn(20000)
cacheFile := filepath.Join(t.TempDir(), "cache.db")
configFile := newEmptyFile(t)
done := make(chan error, 1)
go func() {
app, _, _, _ := newTestApp()
done <- app.Run([]string{"ntfy", "serve", "--config=" + configFile, fmt.Sprintf("--listen-http=:%d", port),
"--cache-file=" + cacheFile, "--cache-batch-size=100", "--cache-batch-timeout=1h"})
}()
url := fmt.Sprintf("http://127.0.0.1:%d/mytopic", port)
var resp *http.Response
var err error
for i := 0; i < 40; i++ {
if resp, err = http.Post(url, "text/plain", strings.NewReader("batched")); err == nil {
break
}
time.Sleep(50 * time.Millisecond)
}
require.Nil(t, err)
require.Equal(t, 200, resp.StatusCode)
require.Nil(t, syscall.Kill(syscall.Getpid(), syscall.SIGTERM))
select {
case err := <-done:
require.Nil(t, err)
case <-time.After(10 * time.Second):
t.Fatal("serve did not exit after SIGTERM")
}
cache, err := message.NewSQLiteStore(cacheFile, "", time.Hour, 0, 0, false)
require.Nil(t, err)
defer cache.Close()
messages, err := cache.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, 1, len(messages))
}
cmd/serve_windows.go
+5
View File
@@ -19,6 +19,11 @@ func sigHandlerConfigReload(config string) {
log.Debug("Config hot-reload via SIGHUP is not supported on Windows")
}
// sigHandlerShutdown is a no-op on Windows; the service control manager stops the server
// via windowsService.Execute
func sigHandlerShutdown(s *server.Server, stopping chan<- struct{}) {
}
// runAsWindowsService runs the ntfy server as a Windows service
func runAsWindowsService(conf *server.Config) error {
return svc.Run(serviceName, &windowsService{conf: conf})
db/db.go
+29
View File
@@ -57,6 +57,16 @@ func (d *DB) Exec(query string, args ...any) (sql.Result, error) {
return d.primary.DB.Exec(query, args...)
}
// ExecContext delegates to the primary database.
func (d *DB) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) {
return d.primary.DB.ExecContext(ctx, query, args...)
}
// QueryContext delegates to the primary database.
func (d *DB) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) {
return d.primary.DB.QueryContext(ctx, query, args...)
}
// Begin delegates to the primary database.
func (d *DB) Begin() (*sql.Tx, error) {
return d.primary.DB.Begin()
@@ -73,6 +83,25 @@ func (d *DB) Primary() *sql.DB {
return d.primary.DB
}
// Stats returns the connection pool statistics of the primary, then of each replica
func (d *DB) Stats() []*PoolStats {
stats := []*PoolStats{
{
Role: PoolRolePrimary,
Addr: d.primary.Addr,
Stats: d.primary.DB.Stats(),
},
}
for _, r := range d.replicas {
stats = append(stats, &PoolStats{
Role: PoolRoleReplica,
Addr: r.Addr,
Stats: r.DB.Stats(),
})
}
return stats
}
// ReadOnly returns a *sql.DB suitable for read-only queries. It round-robins across healthy
// replicas. If all replicas are unhealthy or none are configured, the primary is returned.
func (d *DB) ReadOnly() *sql.DB {
db/db_test.go
+76
View File
@@ -0,0 +1,76 @@
package db_test
import (
"context"
"database/sql"
"path/filepath"
"testing"
_ "github.com/mattn/go-sqlite3"
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/db"
)
func TestDB_ExecContextAndQueryContext(t *testing.T) {
sqlite, err := sql.Open("sqlite3", filepath.Join(t.TempDir(), "test.db"))
require.Nil(t, err)
d := db.New(&db.Host{DB: sqlite}, nil)
defer d.Close()
ctx := context.Background()
_, err = d.ExecContext(ctx, "CREATE TABLE things (id INT)")
require.Nil(t, err)
_, err = d.ExecContext(ctx, "INSERT INTO things VALUES (1)")
require.Nil(t, err)
rows, err := d.QueryContext(ctx, "SELECT id FROM things")
require.Nil(t, err)
defer rows.Close()
require.True(t, rows.Next())
var id int
require.Nil(t, rows.Scan(&id))
require.Equal(t, 1, id)
// The context is passed through, so a cancelled one is honored
cancelled, cancel := context.WithCancel(ctx)
cancel()
_, err = d.ExecContext(cancelled, "INSERT INTO things VALUES (2)")
require.ErrorIs(t, err, context.Canceled)
_, err = d.QueryContext(cancelled, "SELECT id FROM things")
require.ErrorIs(t, err, context.Canceled)
}
func TestDB_Stats(t *testing.T) {
// One entry per pool, primary first, so the server can export each pool's connection counts
primary, err := sql.Open("sqlite3", filepath.Join(t.TempDir(), "primary.db"))
require.Nil(t, err)
primary.SetMaxOpenConns(7)
replica, err := sql.Open("sqlite3", filepath.Join(t.TempDir(), "replica.db"))
require.Nil(t, err)
replica.SetMaxOpenConns(3)
d := db.New(&db.Host{
Addr: "primary:5432",
DB: primary,
}, []*db.Host{
{
Addr: "replica:5432",
DB: replica,
},
})
defer d.Close()
// Hold one primary connection, so it shows as in use
conn, err := primary.Conn(context.Background())
require.Nil(t, err)
defer conn.Close()
stats := d.Stats()
require.Len(t, stats, 2)
require.Equal(t, db.PoolRolePrimary, stats[0].Role)
require.Equal(t, "primary:5432", stats[0].Addr)
require.Equal(t, 7, stats[0].Stats.MaxOpenConnections)
require.Equal(t, 1, stats[0].Stats.InUse)
require.Equal(t, db.PoolRoleReplica, stats[1].Role)
require.Equal(t, "replica:5432", stats[1].Addr)
require.Equal(t, 3, stats[1].Stats.MaxOpenConnections)
require.Equal(t, 0, stats[1].Stats.InUse)
}
db/pg/pg.go
+7 -3
View File
@@ -39,7 +39,8 @@ func OpenReplica(dsn string) (*db.Host, error) {
}
// open opens a PostgreSQL database connection pool from a DSN string. It supports custom
// query parameters for pool configuration: pool_max_conns (default 10), pool_max_idle_conns,
// query parameters for pool configuration: pool_max_conns (default 10), pool_max_idle_conns
// (default: pool_max_conns),
// pool_conn_max_lifetime, and pool_conn_max_idle_time. These parameters are stripped from
// the DSN before passing it to the driver.
func open(dsn string) (*db.Host, error) {
@@ -76,9 +77,12 @@ func open(dsn string) (*db.Host, error) {
return nil, err
}
d.SetMaxOpenConns(maxOpenConns)
if maxIdleConns > 0 {
d.SetMaxIdleConns(maxIdleConns)
if maxIdleConns == 0 {
// database/sql keeps only 2 idle connections by default, so every burst above that
// would close and re-open connections (DNS + TCP + TLS + auth per query)
maxIdleConns = maxOpenConns
}
d.SetMaxIdleConns(maxIdleConns)
if connMaxLifetime > 0 {
d.SetConnMaxLifetime(connMaxLifetime)
}
db/pg/pg_test.go
+32
View File
@@ -2,6 +2,8 @@ package pg
import (
"net/url"
"os"
"sync"
"testing"
"github.com/stretchr/testify/require"
@@ -51,3 +53,33 @@ func TestCensorPassword(t *testing.T) {
})
}
}
func TestOpen_KeepsPoolConnectionsIdle(t *testing.T) {
// database/sql keeps only 2 idle connections by default, so every burst above that closed
// and re-opened connections (DNS lookup + TCP + TLS + auth per query). Unless configured
// otherwise, the pool now keeps as many idle connections as it may open.
dsn := os.Getenv("NTFY_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("NTFY_TEST_DATABASE_URL not set")
}
u, err := url.Parse(dsn)
require.Nil(t, err)
q := u.Query()
q.Set("pool_max_conns", "8")
u.RawQuery = q.Encode()
host, err := Open(u.String())
require.Nil(t, err)
defer host.DB.Close()
var wg sync.WaitGroup
for i := 0; i < 6; i++ {
wg.Add(1)
go func() {
defer wg.Done()
_, err := host.DB.Exec("SELECT pg_sleep(0.2)")
require.Nil(t, err)
}()
}
wg.Wait()
require.Equal(t, 6, host.DB.Stats().Idle)
}
db/schema/schema.go
+110 -30
View File
@@ -2,12 +2,26 @@
// store's schema inside a single transaction. On PostgreSQL, all stores share one database, so
// versions live in a shared schema_version table keyed by store name. On SQLite, every store is
// its own database file, so the version lives in the schemaVersion table keyed by id = 1.
//
// An older version of ntfy runs on a newer schema (after a rollback, or as an old node during a
// rolling cluster upgrade), so migrations must be additive: new tables, indexes, and columns that
// are nullable or have a default. Never drop or rename a column, add a NOT NULL column without a
// default, or change what existing data means. A change that cannot be additive takes two
// releases: the first adds the new shape and keeps the old one working, a later one removes the old.
//
// If a breaking migration is unavoidable, mark its step with Breaking: the store's minimum version
// (the oldest schema version whose code still runs on the schema) becomes the step's result, and
// older code then refuses to start instead of misreading the data. The minimum is only checked
// when a store starts: nodes already running older code keep going, and code from before the
// minimum existed never checks it. So a breaking migration also needs a deploy that stops or
// upgrades every node running older code first; it cannot be rolled out node by node.
package schema
import (
"database/sql"
"errors"
"fmt"
"math"
"heckel.io/ntfy/v2/db/pg"
"heckel.io/ntfy/v2/log"
@@ -18,22 +32,32 @@ const (
)
const (
sqliteCreateVersionTableQuery = `CREATE TABLE IF NOT EXISTS schemaVersion (id INT PRIMARY KEY, version INT NOT NULL)`
sqliteSelectVersionQuery = `SELECT version FROM schemaVersion WHERE id = 1`
sqliteUpsertVersionQuery = `INSERT INTO schemaVersion (id, version) VALUES (1, ?) ON CONFLICT (id) DO UPDATE SET version = excluded.version`
sqliteCreateVersionTableQuery = `CREATE TABLE IF NOT EXISTS schemaVersion (id INT PRIMARY KEY, version INT NOT NULL, min_version INT NOT NULL DEFAULT 1)`
sqliteHasMinVersionQuery = `SELECT COUNT(*) FROM pragma_table_info('schemaVersion') WHERE name = 'min_version'`
sqliteAddMinVersionQuery = `ALTER TABLE schemaVersion ADD COLUMN min_version INT NOT NULL DEFAULT 1`
sqliteSelectVersionQuery = `SELECT version, min_version FROM schemaVersion WHERE id = 1`
sqliteUpsertVersionQuery = `INSERT INTO schemaVersion (id, version, min_version) VALUES (1, ?, ?) ON CONFLICT (id) DO UPDATE SET version = excluded.version, min_version = excluded.min_version`
postgresCreateVersionTableQuery = `CREATE TABLE IF NOT EXISTS schema_version (store TEXT PRIMARY KEY, version INT NOT NULL)`
postgresSelectVersionQuery = `SELECT version FROM schema_version WHERE store = $1`
postgresUpsertVersionQuery = `INSERT INTO schema_version (store, version) VALUES ($1, $2) ON CONFLICT (store) DO UPDATE SET version = EXCLUDED.version`
postgresCreateVersionTableQuery = `CREATE TABLE IF NOT EXISTS schema_version (store TEXT PRIMARY KEY, version INT NOT NULL, min_version INT NOT NULL DEFAULT 1)`
postgresHasMinVersionQuery = `SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'schema_version' AND column_name = 'min_version'`
postgresAddMinVersionQuery = `ALTER TABLE schema_version ADD COLUMN min_version INT NOT NULL DEFAULT 1`
postgresSelectVersionQuery = `SELECT version, min_version FROM schema_version WHERE store = $1`
postgresUpsertVersionQuery = `INSERT INTO schema_version (store, version, min_version) VALUES ($1, $2, $3) ON CONFLICT (store) DO UPDATE SET version = EXCLUDED.version, min_version = EXCLUDED.min_version`
postgresAdvisoryLockQuery = `SELECT pg_advisory_xact_lock($1)` // Transaction-scoped lock to avoid migration races
)
// Migrate creates or upgrades the named store's schema to targetVersion in one transaction, or
// creates a new database using the "create" function.
func Migrate(db *sql.DB, dialect Dialect, store string, targetVersion int, create MigrateFunc, migrations map[int]MigrateFunc) error {
// Migrate creates or upgrades the named store's schema in one transaction, or creates a new
// database using the "create" function. Both versions follow from the steps: the target is the
// version after the last step (see Migrations.targetVersion), the minimum the version after the newest
// breaking step, or 1 (see package doc).
func Migrate(db *sql.DB, dialect Dialect, store string, create MigrateFunc, migrations Migrations) error {
if dialect != Postgres && dialect != SQLite {
return fmt.Errorf("unsupported schema dialect %d", dialect)
}
targetVersion, err := migrations.targetVersion()
if err != nil {
return fmt.Errorf("invalid %s migration steps: %w", store, err)
}
tx, err := db.Begin()
if err != nil {
return fmt.Errorf("cannot begin %s schema transaction: %w", store, err)
@@ -46,66 +70,122 @@ func Migrate(db *sql.DB, dialect Dialect, store string, targetVersion int, creat
return fmt.Errorf("cannot acquire %s schema advisory lock: %w", store, err)
}
}
if _, err := tx.Exec(createVersionTableQuery(dialect)); err != nil {
return fmt.Errorf("cannot create schema version table: %w", err)
minVersion := migrations.minVersion()
if err := ensureVersionTable(tx, dialect); err != nil {
return err
}
version, err := readVersion(tx, dialect, store)
version, storedMinVersion, err := readVersion(tx, dialect, store)
if errors.Is(err, sql.ErrNoRows) {
// Fresh database: create the store's tables at the target version
if err := create(tx); err != nil {
return fmt.Errorf("cannot create %s schema: %w", store, err)
}
if err := writeVersion(tx, dialect, store, targetVersion); err != nil {
if err := writeVersion(tx, dialect, store, targetVersion, minVersion); err != nil {
return fmt.Errorf("cannot write %s schema version: %w", store, err)
}
return tx.Commit()
} else if err != nil {
return fmt.Errorf("cannot read %s schema version: %w", store, err)
}
if version == targetVersion {
if storedMinVersion > targetVersion {
return fmt.Errorf("the %s database schema version %d needs ntfy that supports at least version %d, this version supports up to %d", store, version, storedMinVersion, targetVersion)
} else if version == targetVersion {
return tx.Commit()
} else if version > targetVersion {
// Migrations are additive (see package doc), so this version still works on the newer schema
log.Tag(tag).Warn("The %s database schema version %d is newer than this version of ntfy supports (%d); continuing, since schema changes are backwards compatible", store, version, targetVersion)
return tx.Commit()
}
if version > targetVersion {
return fmt.Errorf("unexpected %s schema version %d, this version of ntfy supports up to %d", store, version, targetVersion)
}
for v := version; v < targetVersion; v++ {
migrate, ok := migrations[v]
step, ok := migrations[MigrationKey{From: v, To: v + 1}]
if !ok {
return fmt.Errorf("cannot find %s migration step from version %d to %d", store, v, v+1)
}
log.Tag(tag).Info("Migrating %s database schema: from %d to %d", store, v, v+1)
if err := migrate(tx); err != nil {
if err := step.Migrate(tx); err != nil {
return fmt.Errorf("%s migration step from version %d to %d failed: %w", store, v, v+1, err)
}
}
if err := writeVersion(tx, dialect, store, targetVersion); err != nil {
if err := writeVersion(tx, dialect, store, targetVersion, minVersion); err != nil {
return fmt.Errorf("cannot write %s schema version: %w", store, err)
}
return tx.Commit()
}
func createVersionTableQuery(dialect Dialect) string {
if dialect == Postgres {
return postgresCreateVersionTableQuery
// targetVersion returns the schema version the steps lead to: the last step's To, or 1
// without steps. Steps go one version at a time and must be contiguous, so a forgotten or
// mis-keyed one fails at startup.
func (m Migrations) targetVersion() (int, error) {
if len(m) == 0 {
return 1, nil
}
return sqliteCreateVersionTableQuery
first, last := math.MaxInt, 0
for k := range m {
if k.To != k.From+1 {
return 0, fmt.Errorf("migration step from version %d to %d is invalid, a step goes one version at a time", k.From, k.To)
}
first, last = min(first, k.From), max(last, k.To)
}
if first < 1 {
return 0, fmt.Errorf("migration step from version %d is invalid, steps start at version 1", first)
}
for v := first; v < last; v++ {
if _, ok := m[MigrationKey{From: v, To: v + 1}]; !ok {
return 0, fmt.Errorf("missing migration step from version %d to %d", v, v+1)
}
}
return last, nil
}
func readVersion(tx *sql.Tx, dialect Dialect, store string) (version int, err error) {
// minVersion returns the oldest schema version whose code still runs on the schema the steps
// lead to: the To of the newest breaking step, or 1 if no step is breaking
func (m Migrations) minVersion() int {
minVersion := 1
for k, step := range m {
if step.Breaking && k.To > minVersion {
minVersion = k.To
}
}
return minVersion
}
// ensureVersionTable creates the version table, and adds the min_version column to one created
// before it existed. Older code writes only the version and leaves the column alone.
func ensureVersionTable(tx *sql.Tx, dialect Dialect) error {
createQuery, hasQuery, addQuery := sqliteCreateVersionTableQuery, sqliteHasMinVersionQuery, sqliteAddMinVersionQuery
if dialect == Postgres {
err = tx.QueryRow(postgresSelectVersionQuery, store).Scan(&version)
createQuery, hasQuery, addQuery = postgresCreateVersionTableQuery, postgresHasMinVersionQuery, postgresAddMinVersionQuery
}
if _, err := tx.Exec(createQuery); err != nil {
return fmt.Errorf("cannot create schema version table: %w", err)
}
var columns int
if err := tx.QueryRow(hasQuery).Scan(&columns); err != nil {
return fmt.Errorf("cannot inspect schema version table: %w", err)
} else if columns > 0 {
return nil
}
if _, err := tx.Exec(addQuery); err != nil {
return fmt.Errorf("cannot add min_version to schema version table: %w", err)
}
return nil
}
func readVersion(tx *sql.Tx, dialect Dialect, store string) (version, minVersion int, err error) {
if dialect == Postgres {
err = tx.QueryRow(postgresSelectVersionQuery, store).Scan(&version, &minVersion)
} else {
err = tx.QueryRow(sqliteSelectVersionQuery).Scan(&version)
err = tx.QueryRow(sqliteSelectVersionQuery).Scan(&version, &minVersion)
}
return
}
func writeVersion(tx *sql.Tx, dialect Dialect, store string, version int) error {
func writeVersion(tx *sql.Tx, dialect Dialect, store string, version, minVersion int) error {
var err error
if dialect == Postgres {
_, err = tx.Exec(postgresUpsertVersionQuery, store, version)
_, err = tx.Exec(postgresUpsertVersionQuery, store, version, minVersion)
} else {
_, err = tx.Exec(sqliteUpsertVersionQuery, version)
_, err = tx.Exec(sqliteUpsertVersionQuery, version, minVersion)
}
return err
}
db/schema/schema_internal_test.go
+61
View File
@@ -0,0 +1,61 @@
package schema
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestMigrations_MinVersion(t *testing.T) {
breaking := Breaking(Nop)
cases := []struct {
name string
migrations Migrations
expected int
}{
{"no steps", Migrations{}, 1},
{"only additive steps", Migrations{{From: 1, To: 2}: Nop, {From: 2, To: 3}: Nop, {From: 3, To: 4}: Nop}, 1},
{"one breaking step", Migrations{{From: 1, To: 2}: Nop, {From: 2, To: 3}: Nop, {From: 3, To: 4}: breaking, {From: 4, To: 5}: Nop, {From: 5, To: 6}: Nop}, 4},
{"newest breaking step wins", Migrations{{From: 1, To: 2}: Nop, {From: 2, To: 3}: breaking, {From: 3, To: 4}: Nop, {From: 4, To: 5}: Nop, {From: 5, To: 6}: breaking, {From: 6, To: 7}: Nop}, 6},
{"breaking last step", Migrations{{From: 4, To: 5}: Nop, {From: 5, To: 6}: breaking}, 6},
{"breaking first step", Migrations{{From: 1, To: 2}: breaking, {From: 2, To: 3}: Nop}, 2},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
require.Equal(t, c.expected, c.migrations.minVersion())
})
}
}
func TestMigrations_TargetVersion(t *testing.T) {
cases := []struct {
name string
migrations Migrations
expected int
}{
{"no steps", nil, 1},
{"one step", Migrations{{From: 1, To: 2}: Nop}, 2},
{"steps starting later", Migrations{{From: 14, To: 15}: Nop, {From: 15, To: 16}: Nop, {From: 16, To: 17}: Nop}, 17}, // A store added to Postgres at version 14
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
version, err := c.migrations.targetVersion()
require.Nil(t, err)
require.Equal(t, c.expected, version)
})
}
}
func TestMigrations_TargetVersion_Invalid(t *testing.T) {
_, err := Migrations{{From: 1, To: 2}: Nop, {From: 3, To: 4}: Nop, {From: 4, To: 5}: Nop}.targetVersion()
require.ErrorContains(t, err, "missing migration step from version 2 to 3")
_, err = Migrations{{From: 0, To: 1}: Nop, {From: 1, To: 2}: Nop}.targetVersion()
require.ErrorContains(t, err, "migration step from version 0 is invalid")
// A step goes exactly one version: a key skipping one would make fresh installs record a
// version that existing installs never reach
_, err = Migrations{{From: 1, To: 2}: Nop, {From: 2, To: 4}: Nop}.targetVersion()
require.ErrorContains(t, err, "migration step from version 2 to 4 is invalid, a step goes one version at a time")
_, err = Migrations{{From: 3, To: 3}: Nop}.targetVersion()
require.ErrorContains(t, err, "migration step from version 3 to 3 is invalid, a step goes one version at a time")
}
db/schema/schema_test.go
+215 -30
View File
@@ -1,8 +1,10 @@
package schema_test
import (
"bytes"
"database/sql"
"fmt"
"os"
"path/filepath"
"testing"
@@ -10,6 +12,7 @@ import (
"heckel.io/ntfy/v2/db/pg"
"heckel.io/ntfy/v2/db/schema"
dbtest "heckel.io/ntfy/v2/db/test"
"heckel.io/ntfy/v2/log"
_ "github.com/mattn/go-sqlite3"
)
@@ -49,49 +52,76 @@ func forEachDialect(t *testing.T, f func(t *testing.T, d *sql.DB, dialect schema
func TestMigrate_FreshCreate(t *testing.T) {
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
// A fresh database jumps straight to the target version; migration steps are not consulted
require.Nil(t, schema.Migrate(d, dialect, "things", 3, testCreate, nil))
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, stepsTo(3)))
_, err := d.Exec(`INSERT INTO things (id, name) VALUES ('a', 'thing a')`)
require.Nil(t, err)
require.Equal(t, 3, storeVersion(t, d, dialect, "things"))
// Idempotent: a second node boots against the migrated schema
require.Nil(t, schema.Migrate(d, dialect, "things", 3, testCreate, nil))
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, stepsTo(3)))
})
}
func TestMigrate_AppliesMigrationSteps(t *testing.T) {
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
require.Nil(t, schema.Migrate(d, dialect, "things", 1, testCreate, nil))
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, nil))
// A newer version of the code migrates 1 -> 3 step by step, in order
migrations := map[int]schema.MigrateFunc{
1: func(tx *sql.Tx) error {
migrations := schema.Migrations{
{From: 1, To: 2}: schema.Func(func(tx *sql.Tx) error {
_, err := tx.Exec(`ALTER TABLE things ADD COLUMN color TEXT NOT NULL DEFAULT ''`)
return err
},
2: func(tx *sql.Tx) error {
}),
{From: 2, To: 3}: schema.Func(func(tx *sql.Tx) error {
_, err := tx.Exec(`ALTER TABLE things ADD COLUMN size INT NOT NULL DEFAULT 0`)
return err
},
}),
}
require.Nil(t, schema.Migrate(d, dialect, "things", 3, testCreate, migrations))
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, migrations))
_, err := d.Exec(`INSERT INTO things (id, name, color, size) VALUES ('b', 'thing b', 'red', 2)`)
require.Nil(t, err)
require.Equal(t, 3, storeVersion(t, d, dialect, "things"))
})
}
func TestMigrate_TargetVersionFollowsSteps(t *testing.T) {
// The target version is the one after the last step: adding a step is all it takes, there is
// no separate version number to bump (and to forget)
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, nil))
require.Equal(t, 1, storeVersion(t, d, dialect, "things"))
migrations := schema.Migrations{
{From: 1, To: 2}: schema.Query(`ALTER TABLE things ADD COLUMN color TEXT NOT NULL DEFAULT ''`),
{From: 2, To: 3}: schema.Query(`ALTER TABLE things ADD COLUMN size INT NOT NULL DEFAULT 0`),
}
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, migrations))
require.Equal(t, 3, storeVersion(t, d, dialect, "things"))
_, err := d.Exec(`INSERT INTO things (id, name, color, size) VALUES ('a', 'thing a', 'red', 2)`)
require.Nil(t, err)
})
}
func TestMigrate_GapInStepsFails(t *testing.T) {
// A missing step in the middle fails before anything is created or migrated
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
err := schema.Migrate(d, dialect, "things", testCreate, schema.Migrations{{From: 1, To: 2}: schema.Nop, {From: 3, To: 4}: schema.Nop})
require.ErrorContains(t, err, "missing migration step from version 2 to 3")
_, err = d.Exec(`INSERT INTO things (id, name) VALUES ('a', 'thing a')`)
require.Error(t, err) // Not created
})
}
func TestMigrate_ClosureCarriesConfig(t *testing.T) {
// Migrations needing config take it via closure at map-construction time; there is no
// params plumbing in the framework itself
migrationsFor := func(defaultName string) map[int]schema.MigrateFunc {
return map[int]schema.MigrateFunc{
1: schema.AsMigrateFunc(fmt.Sprintf(`ALTER TABLE things ADD COLUMN nick TEXT NOT NULL DEFAULT '%s'`, defaultName)),
migrationsFor := func(defaultName string) schema.Migrations {
return schema.Migrations{
{From: 1, To: 2}: schema.Query(fmt.Sprintf(`ALTER TABLE things ADD COLUMN nick TEXT NOT NULL DEFAULT '%s'`, defaultName)),
}
}
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
require.Nil(t, schema.Migrate(d, dialect, "things", 1, testCreate, nil))
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, nil))
_, err := d.Exec(`INSERT INTO things (id, name) VALUES ('a', 'thing a')`)
require.Nil(t, err)
require.Nil(t, schema.Migrate(d, dialect, "things", 2, testCreate, migrationsFor("configured-default")))
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, migrationsFor("configured-default")))
var nick string
require.Nil(t, d.QueryRow(`SELECT nick FROM things WHERE id = 'a'`).Scan(&nick))
require.Equal(t, "configured-default", nick)
@@ -100,23 +130,130 @@ func TestMigrate_ClosureCarriesConfig(t *testing.T) {
func TestMigrate_InvalidDialect(t *testing.T) {
d := openTestSQLite(t)
err := schema.Migrate(d, schema.Dialect(99), "things", 1, testCreate, nil)
err := schema.Migrate(d, schema.Dialect(99), "things", testCreate, nil)
require.Error(t, err)
}
func TestMigrate_RefusesFutureVersion(t *testing.T) {
func TestMigrate_NewerVersionWarnsAndStarts(t *testing.T) {
// An older version of ntfy (a rollback, or an old node during a rolling upgrade) runs on a
// newer schema: it warns, starts, and leaves the newer version in place
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
require.Nil(t, schema.Migrate(d, dialect, "things", 2, testCreate, map[int]schema.MigrateFunc{}))
err := schema.Migrate(d, dialect, "things", 1, testCreate, nil)
require.Error(t, err)
var out bytes.Buffer
log.SetOutput(&out)
t.Cleanup(func() { log.SetOutput(os.Stderr) })
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, stepsTo(2)))
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, nil))
require.Contains(t, out.String(), "things database schema version 2 is newer than this version of ntfy supports (1)")
require.Equal(t, 2, storeVersion(t, d, dialect, "things"))
})
}
func TestMigrate_BreakingNewerSchemaFails(t *testing.T) {
// A breaking migration raises the store's minimum version: older code must not run on it
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, schema.Migrations{{From: 1, To: 2}: schema.Nop, {From: 2, To: 3}: schema.Breaking(schema.Nop)}))
err := schema.Migrate(d, dialect, "things", testCreate, stepsTo(2))
require.ErrorContains(t, err, "things database schema version 3 needs ntfy that supports at least version 3, this version supports up to 2")
})
}
func TestMigrate_BreakingStepRaisesMinVersion(t *testing.T) {
// The minimum version follows from the steps: the version after the newest breaking step
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, nil))
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, schema.Migrations{
{From: 1, To: 2}: schema.Breaking(schema.Nop),
{From: 2, To: 3}: schema.Nop,
}))
require.Equal(t, 2, storeMinVersion(t, d, dialect, "things"))
})
}
func TestMigrate_FreshCreateTakesMinVersionFromSteps(t *testing.T) {
// A fresh database is created at the target version directly, but its minimum still comes from the steps
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, schema.Migrations{
{From: 1, To: 2}: schema.Nop,
{From: 2, To: 3}: schema.Breaking(schema.Nop),
}))
require.Equal(t, 3, storeMinVersion(t, d, dialect, "things"))
})
}
func TestMigrate_AddsMinVersionToAnExistingVersionTable(t *testing.T) {
// Version tables created before the minimum version existed get the column; older code,
// which writes only the version, keeps working on the new table
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
createOld, upsertOld := `CREATE TABLE schema_version (store TEXT PRIMARY KEY, version INT NOT NULL)`, `INSERT INTO schema_version (store, version) VALUES ('things', 1) ON CONFLICT (store) DO UPDATE SET version = EXCLUDED.version`
if dialect == schema.SQLite {
createOld, upsertOld = `CREATE TABLE schemaVersion (id INT PRIMARY KEY, version INT NOT NULL)`, `INSERT INTO schemaVersion (id, version) VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET version = excluded.version`
}
_, err := d.Exec(testCreateQuery)
require.Nil(t, err)
_, err = d.Exec(createOld)
require.Nil(t, err)
_, err = d.Exec(upsertOld)
require.Nil(t, err)
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, nil))
require.Equal(t, 1, storeMinVersion(t, d, dialect, "things"))
_, err = d.Exec(upsertOld) // Older code writing its version
require.Nil(t, err)
// A breaking migration to version 2 locks out code that supports only version 1
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, schema.Migrations{{From: 1, To: 2}: schema.Breaking(schema.Nop)}))
require.Equal(t, 2, storeMinVersion(t, d, dialect, "things"))
require.Error(t, schema.Migrate(d, dialect, "things", testCreate, nil))
})
}
func TestMigrate_VersionTableCreatedWhenMissing(t *testing.T) {
// A database without a version table gets one with both columns, filled from the steps
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
require.Empty(t, versionTableColumns(t, d, dialect))
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, schema.Migrations{{From: 1, To: 2}: schema.Nop, {From: 2, To: 3}: schema.Breaking(schema.Nop)}))
require.ElementsMatch(t, []string{versionTableKeyColumn(dialect), "version", "min_version"}, versionTableColumns(t, d, dialect))
require.Equal(t, 3, storeVersion(t, d, dialect, "things"))
require.Equal(t, 3, storeMinVersion(t, d, dialect, "things"))
})
}
func TestMigrate_OldVersionTableGetsMinVersion(t *testing.T) {
// A version table from before min_version (ntfy <= v2.31) gets the column; the stored version
// is kept, and the minimum defaults to 1
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
createOld, insertOld := `CREATE TABLE schema_version (store TEXT PRIMARY KEY, version INT NOT NULL)`, `INSERT INTO schema_version (store, version) VALUES ('things', 2)`
if dialect == schema.SQLite {
createOld, insertOld = `CREATE TABLE schemaVersion (id INT PRIMARY KEY, version INT NOT NULL)`, `INSERT INTO schemaVersion (id, version) VALUES (1, 2)`
}
for _, query := range []string{testCreateQuery, createOld, insertOld} {
_, err := d.Exec(query)
require.Nil(t, err)
}
require.ElementsMatch(t, []string{versionTableKeyColumn(dialect), "version"}, versionTableColumns(t, d, dialect))
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, stepsTo(2)))
require.ElementsMatch(t, []string{versionTableKeyColumn(dialect), "version", "min_version"}, versionTableColumns(t, d, dialect))
require.Equal(t, 2, storeVersion(t, d, dialect, "things"))
require.Equal(t, 1, storeMinVersion(t, d, dialect, "things"))
})
}
func TestMigrate_CurrentVersionTableIsLeftAlone(t *testing.T) {
// A version table that already has min_version is not altered again, and its values stay
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
migrations := schema.Migrations{{From: 1, To: 2}: schema.Breaking(schema.Nop)}
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, migrations))
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, migrations))
require.ElementsMatch(t, []string{versionTableKeyColumn(dialect), "version", "min_version"}, versionTableColumns(t, d, dialect))
require.Equal(t, 2, storeVersion(t, d, dialect, "things"))
require.Equal(t, 2, storeMinVersion(t, d, dialect, "things"))
})
}
func TestMigrate_MissingStepFails(t *testing.T) {
forEachDialect(t, func(t *testing.T, d *sql.DB, dialect schema.Dialect) {
require.Nil(t, schema.Migrate(d, dialect, "things", 1, testCreate, nil))
err := schema.Migrate(d, dialect, "things", 3, testCreate, nil) // No step 1 -> 2 registered
require.Error(t, err)
require.Nil(t, schema.Migrate(d, dialect, "things", testCreate, nil))
err := schema.Migrate(d, dialect, "things", testCreate, schema.Migrations{{From: 2, To: 3}: schema.Nop}) // No step 1 -> 2 registered
require.ErrorContains(t, err, "cannot find things migration step from version 1 to 2")
})
}
@@ -124,11 +261,11 @@ func TestMigrate_StoresAreIndependent(t *testing.T) {
// Postgres only: stores share one database, tracked as rows in schema_version. On SQLite
// every store has its own database file, so independence is by file.
d := openTestPostgres(t)
require.Nil(t, schema.Migrate(d, schema.Postgres, "things", 1, testCreate, nil))
require.Nil(t, schema.Migrate(d, schema.Postgres, "gadgets", 4, func(tx *sql.Tx) error {
require.Nil(t, schema.Migrate(d, schema.Postgres, "things", testCreate, nil))
require.Nil(t, schema.Migrate(d, schema.Postgres, "gadgets", func(tx *sql.Tx) error {
_, err := tx.Exec(`CREATE TABLE IF NOT EXISTS gadgets (id TEXT PRIMARY KEY)`)
return err
}, nil))
}, stepsTo(4)))
require.Equal(t, 1, storeVersion(t, d, schema.Postgres, "things"))
require.Equal(t, 4, storeVersion(t, d, schema.Postgres, "gadgets"))
}
@@ -144,13 +281,13 @@ func TestMigrate_SQLiteReadsExistingSchemaVersionTable(t *testing.T) {
require.Nil(t, err)
_, err = d.Exec(`INSERT INTO schemaVersion VALUES (1, 1)`)
require.Nil(t, err)
migrations := map[int]schema.MigrateFunc{
1: func(tx *sql.Tx) error {
migrations := schema.Migrations{
{From: 1, To: 2}: schema.Func(func(tx *sql.Tx) error {
_, err := tx.Exec(`ALTER TABLE things ADD COLUMN color TEXT NOT NULL DEFAULT ''`)
return err
},
}),
}
require.Nil(t, schema.Migrate(d, schema.SQLite, "things", 2, testCreate, migrations))
require.Nil(t, schema.Migrate(d, schema.SQLite, "things", testCreate, migrations))
_, err = d.Exec(`INSERT INTO things (id, name, color) VALUES ('a', 'thing a', 'red')`)
require.Nil(t, err)
require.Equal(t, 2, storeVersion(t, d, schema.SQLite, "things"))
@@ -170,7 +307,7 @@ func TestMigrate_ConcurrentFreshCreate(t *testing.T) {
return
}
defer host.DB.Close()
errs <- schema.Migrate(host.DB, schema.Postgres, "things", 1, testCreate, nil)
errs <- schema.Migrate(host.DB, schema.Postgres, "things", testCreate, nil)
}()
}
for i := 0; i < n; i++ {
@@ -178,6 +315,17 @@ func TestMigrate_ConcurrentFreshCreate(t *testing.T) {
}
}
func storeMinVersion(t *testing.T, d *sql.DB, dialect schema.Dialect, store string) int {
t.Helper()
var minVersion int
if dialect == schema.Postgres {
require.Nil(t, d.QueryRow(`SELECT min_version FROM schema_version WHERE store = $1`, store).Scan(&minVersion))
} else {
require.Nil(t, d.QueryRow(`SELECT min_version FROM schemaVersion WHERE id = 1`).Scan(&minVersion))
}
return minVersion
}
func storeVersion(t *testing.T, d *sql.DB, dialect schema.Dialect, store string) int {
t.Helper()
var version int
@@ -188,3 +336,40 @@ func storeVersion(t *testing.T, d *sql.DB, dialect schema.Dialect, store string)
}
return version
}
// versionTableColumns returns the version table's column names, or none if it does not exist
func versionTableColumns(t *testing.T, d *sql.DB, dialect schema.Dialect) []string {
t.Helper()
query := `SELECT column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'schema_version'`
if dialect == schema.SQLite {
query = `SELECT name FROM pragma_table_info('schemaVersion')`
}
rows, err := d.Query(query)
require.Nil(t, err)
defer rows.Close()
columns := make([]string, 0)
for rows.Next() {
var column string
require.Nil(t, rows.Scan(&column))
columns = append(columns, column)
}
require.Nil(t, rows.Err())
return columns
}
// versionTableKeyColumn returns the version table's key: one row per store on Postgres, one row on SQLite
func versionTableKeyColumn(dialect schema.Dialect) string {
if dialect == schema.Postgres {
return "store"
}
return "id"
}
// stepsTo returns no-op steps leading to the given schema version
func stepsTo(version int) schema.Migrations {
migrations := make(schema.Migrations)
for v := 1; v < version; v++ {
migrations[schema.MigrationKey{From: v, To: v + 1}] = schema.Nop
}
return migrations
}
db/schema/types.go
+36 -4
View File
@@ -13,7 +13,7 @@ const (
// MigrateFunc applies one schema change inside the setup transaction: the initial creation of
// a store's tables, or one step upgrading a store from version N to N+1. Migrations needing
// config capture it via closure, e.g. func migrations(cacheDuration time.Duration) map[int]MigrateFunc.
// config capture it via closure, e.g. func migrations(cacheDuration time.Duration) Migrations.
type MigrateFunc func(tx *sql.Tx) error
// AsMigrateFunc converts a simple query to a migration function
@@ -24,8 +24,40 @@ func AsMigrateFunc(query string) MigrateFunc {
}
}
// NopMigrateFunc is a migration step that does nothing, for versions where a dialect has no
// work to do (e.g. when only the other dialect's schema changed).
func NopMigrateFunc(_ *sql.Tx) error {
// MigrationKey names a migration step by the versions it upgrades between; To is always From+1
type MigrationKey struct {
From int
To int
}
// Migration is one step upgrading a store from version N to N+1
type Migration struct {
Migrate MigrateFunc
Breaking bool // Older code cannot run on the result (see package doc): the store's minimum version becomes the step's To
}
// Migrations are a store's migration steps, keyed by the versions each upgrades between. Steps go
// one version at a time without gaps, which Migrate checks at startup.
type Migrations map[MigrationKey]Migration
// Nop is a migration step that does nothing, for versions where a dialect has no work to do (e.g.
// when only the other dialect's schema changed)
var Nop = Func(func(_ *sql.Tx) error {
return nil
})
// Query is an additive migration step running a query
func Query(query string) Migration {
return Func(AsMigrateFunc(query))
}
// Func is an additive migration step running a function
func Func(f MigrateFunc) Migration {
return Migration{Migrate: f}
}
// Breaking marks a migration step as breaking: older code cannot run on its result (see package doc)
func Breaking(m Migration) Migration {
m.Breaking = true
return m
}
db/test/test.go
+103 -2
View File
@@ -2,9 +2,13 @@ package dbtest
import (
"fmt"
"io"
"net"
"net/url"
"os"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/db"
@@ -17,7 +21,7 @@ const testPoolMaxConns = "2"
// CreateTestPostgresSchema creates a temporary PostgreSQL schema and returns the DSN pointing to it.
// It registers a cleanup function to drop the schema when the test finishes.
// If NTFY_TEST_DATABASE_URL is not set, the test is skipped.
func CreateTestPostgresSchema(t *testing.T) string {
func CreateTestPostgresSchema(t testing.TB) string {
t.Helper()
dsn := os.Getenv("NTFY_TEST_DATABASE_URL")
if dsn == "" {
@@ -51,7 +55,7 @@ func CreateTestPostgresSchema(t *testing.T) string {
// CreateTestPostgres creates a temporary PostgreSQL schema and returns an open *db.DB connection to it.
// It registers cleanup functions to close the DB and drop the schema when the test finishes.
// If NTFY_TEST_DATABASE_URL is not set, the test is skipped.
func CreateTestPostgres(t *testing.T) *db.DB {
func CreateTestPostgres(t testing.TB) *db.DB {
t.Helper()
schemaDSN := CreateTestPostgresSchema(t)
testHost, err := pg.Open(schemaDSN)
@@ -62,3 +66,100 @@ func CreateTestPostgres(t *testing.T) *db.DB {
})
return d
}
// CreateTestPostgresWithLatency is CreateTestPostgres, but every request the client sends reaches
// the database latency later, so each database round trip costs at least latency (as it does
// against a remote managed database). The returned proxy counts the requests, so a test can
// assert how many round trips a code path makes without depending on the speed of the test host.
func CreateTestPostgresWithLatency(t *testing.T, latency time.Duration) (*db.DB, *LatencyProxy) {
t.Helper()
proxy := NewLatencyProxy(t, CreateTestPostgresSchema(t), latency)
testHost, err := pg.Open(proxy.DSN())
require.Nil(t, err)
d := db.New(testHost, nil)
t.Cleanup(func() {
d.Close()
})
return d, proxy
}
// WithLatency starts a LatencyProxy to the database in dsn and returns the DSN pointing at it
func WithLatency(t *testing.T, dsn string, latency time.Duration) string {
t.Helper()
return NewLatencyProxy(t, dsn, latency).DSN()
}
// LatencyProxy is a local TCP proxy to a database that delays everything the client sends by a
// fixed latency and counts the client's requests (a request is one client write, which for the
// pgx driver is one round trip: one query, or one batch of pipelined protocol messages).
type LatencyProxy struct {
dsn string
latency time.Duration
requests atomic.Int64
}
// NewLatencyProxy starts a LatencyProxy to the database in dsn; it stops when the test ends
func NewLatencyProxy(t *testing.T, dsn string, latency time.Duration) *LatencyProxy {
t.Helper()
u, err := url.Parse(dsn)
require.Nil(t, err)
upstream := u.Host
if u.Port() == "" {
upstream = net.JoinHostPort(u.Hostname(), "5432")
}
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.Nil(t, err)
t.Cleanup(func() {
listener.Close()
})
p := &LatencyProxy{latency: latency}
go func() {
for {
client, err := listener.Accept()
if err != nil {
return // Listener closed
}
go p.proxy(client, upstream)
}
}()
u.Host = listener.Addr().String()
p.dsn = u.String()
return p
}
// DSN returns the database URL pointing at the proxy
func (p *LatencyProxy) DSN() string {
return p.dsn
}
// Requests returns the number of client requests forwarded so far
func (p *LatencyProxy) Requests() int64 {
return p.requests.Load()
}
func (p *LatencyProxy) proxy(client net.Conn, upstream string) {
defer client.Close()
server, err := net.Dial("tcp", upstream)
if err != nil {
return
}
defer server.Close()
go func() {
io.Copy(client, server)
client.Close()
}()
buf := make([]byte, 64*1024)
for {
n, err := client.Read(buf)
if n > 0 {
p.requests.Add(1)
time.Sleep(p.latency)
if _, err := server.Write(buf[:n]); err != nil {
return
}
}
if err != nil {
return
}
}
}
db/types.go
+13
View File
@@ -23,3 +23,16 @@ type Host struct {
DB *sql.DB
healthy atomic.Bool
}
// Roles of a connection pool, see PoolStats
const (
PoolRolePrimary = "primary"
PoolRoleReplica = "replica"
)
// PoolStats is one connection pool's statistics: the primary's or a replica's
type PoolStats struct {
Role string // PoolRolePrimary or PoolRoleReplica
Addr string // "host:port"
Stats sql.DBStats
}
docs/ai-policy.md
+27
View File
@@ -0,0 +1,27 @@
# AI policy
I use AI tools to help develop ntfy. Here's how they fit into my work, and what I expect from contributors who use them.
## How I use AI
I use tools like Claude and Codex to write code for ntfy, and **I review every line before it's merged and released**.
If anything, I review code more thoroughly now than I did before. I also write more unit and integration tests, since AI
makes them easier to put together.
For larger changes, I usually read through the diff several times and ask the AI to put together before-and-after
comparisons to help me check the logic. I also run manual regression tests and load tests on a staging server before
releasing those changes on ntfy.sh.
[PostgreSQL support](https://github.com/binwiederhier/ntfy/issues/1114), added in v2.18.0, is a good example. AI wrote the
code, and I spent weeks reviewing and testing it before release. I've shared more about that process in
[this discussion](https://github.com/binwiederhier/ntfy/issues/1645).
AI writes a lot of the code, and I'm still responsible for every line of it.
## AI-assisted pull requests
**AI-assisted pull requests are welcome**, but please **don't make me the first human to read your code**. Before you
submit a pull request, review the code, make sure you understand it, and test it yourself. Don't pass along whatever
your AI tool produced and leave the review to me. **[Don't meat proxy me!](https://nomeatproxy.com/)**
The same applies to issues, pull request descriptions, and comments. Please write them yourself, or at least read and
edit any AI-generated text before posting it. Keep it concise and make sure it says what you actually want to say.
Pull requests that appear to have been generated and submitted without human review will be closed.
docs/config.md
+179 -7
View File
@@ -24,6 +24,10 @@ get a list of [command line options](#command-line-options).
The most basic settings are `base-url` (the external URL of the ntfy server), the HTTP/HTTPS listen address (`listen-http`
and `listen-https`), and socket path (`listen-unix`). All the other things are additional features.
`listen-http` may list several comma-separated addresses, e.g. `10.0.0.5:2586,10.0.0.5:2588`. This is useful behind a
load balancer: it connects to each address from one source IP, so a single address caps it at ~63k connections (one per
source port), and every extra address adds as many again.
Here are a few working sample configs using a `/etc/ntfy/server.yml` file:
=== "server.yml (HTTP-only, with cache + attachments)"
@@ -473,10 +477,29 @@ the primary and replica URLs):
| Parameter | Default | Description |
|---------------------------|---------|----------------------------------------------------------------------------------|
| `pool_max_conns` | 10 | Maximum number of open connections to the database |
| `pool_max_idle_conns` | - | Maximum number of idle connections in the pool |
| `pool_max_idle_conns` | = `pool_max_conns` | Maximum number of idle connections in the pool |
| `pool_conn_max_lifetime` | - | Maximum amount of time a connection may be reused (Go duration, e.g. `5m`, `1h`) |
| `pool_conn_max_idle_time` | - | Maximum amount of time a connection may be idle (Go duration, e.g. `30s`, `5m`) |
For a busy server, the recommended settings are `pool_max_conns=30&pool_conn_max_lifetime=15m&pool_conn_max_idle_time=5m`
on the primary and on every replica (this is what ntfy.sh runs). The reasoning:
* `pool_max_conns`: the message writer uses one connection per batch, and every other query (polls, access checks,
stats) uses one for a few milliseconds, so 30 covers publish bursts of hundreds of messages per second. Make sure the
PostgreSQL `max_connections` setting covers the pool size times the number of ntfy instances, plus other clients.
* `pool_max_idle_conns`: leave it at the default (equal to `pool_max_conns`). A smaller value closes connections after
every burst, so the next burst pays a DNS lookup, a TCP/TLS handshake and authentication per query.
* `pool_conn_max_lifetime`: a long-lived PostgreSQL backend keeps growing (statement and catalog caches are never given
back), and a managed database may move to a new address on failover; recycling connections every 15 minutes bounds
both without noticeable churn.
* `pool_conn_max_idle_time`: releases connections the server does not need overnight; under steady traffic the pool
rotates through all connections, so the limit never triggers.
To size the pools, watch the `ntfy_db_pool_*` [metrics](#monitoring), one series per pool (`role` is `primary` or
`replica`): `ntfy_db_pool_connections_in_use` is how many connections are busy right now, and `ntfy_db_pool_waits_total`
and `ntfy_db_pool_wait_seconds_total` count queries that had to wait for a free connection. The open connection count
says little, since the pool rotates through all of them. Waits that keep going up mean the pool is too small.
## Message cache
If desired, ntfy can temporarily keep notifications in an in-memory or an on-disk cache. Caching messages for a short period
@@ -511,7 +534,8 @@ The following config options are relevant to attachments:
* `attachment-cache-dir` is the cache directory for attached files, or an S3 URL for object storage
* `attachment-total-size-limit` is the size limit of the attachment storage (default: 5G)
* `attachment-file-size-limit` is the per-file attachment size limit (e.g. 300k, 2M, 100M, default: 15M)
* `attachment-expiry-duration` is the duration after which uploaded attachments will be deleted (e.g. 3h, 20h, default: 3h)
* `attachment-expiry-duration` is the duration after which uploaded attachments will be deleted (e.g. 3h, 20h, default: 3h). Attachments never outlive
their message, so this is capped at `cache-duration`; set `cache-duration` at least as high if you want longer-lived attachments.
!!! warning
ntfy takes full control over the attachment directory or S3 bucket. Files that match the message ID format without
@@ -1023,7 +1047,7 @@ you can set the `X-Email` header to [send messages via e-mail](publish.md#e-mail
On ntfy.sh, anonymous email sending was disabled due to abuse. To use the email notification feature,
you must verify your email in the web app's [Account section](https://ntfy.sh/account).
As of today, only SMTP servers with PLAIN auth and STARTLS are supported. To enable e-mail sending, you must set the
As of today, only SMTP servers with PLAIN or LOGIN auth and STARTTLS are supported. To enable e-mail sending, you must set the
following settings:
* `base-url` is the root URL for the ntfy server; this is needed for e-mail footer
@@ -2213,12 +2237,152 @@ ntfy fully supports IPv6, though there are a few things to keep in mind.
The official ntfy.sh server supports IPv6. Check out ntfy.sh's [Ansible repository](https://github.com/binwiederhier/ntfy-ansible) for examples of how to
configure [ntfy](https://github.com/binwiederhier/ntfy-ansible/tree/main/roles/ntfy), [nginx](https://github.com/binwiederhier/ntfy-ansible/tree/main/roles/nginx) and [fail2ban](https://github.com/binwiederhier/ntfy-ansible/tree/main/roles/fail2ban).
## Clustering (experimental)
!!! warning
Clustering is **experimental and not ready for production**. Options, behavior and the database tables it uses may
still change between releases.
ntfy can run as several nodes sharing one PostgreSQL database, so that a message published on one node reaches
subscribers on all of them. It requires **PostgreSQL** as the database backend (`database-url`, see
[database options](#database-options)); the shared database holds all durable state (messages, users, web push) plus
the tables the nodes coordinate through. If you use [attachments](#attachments), the nodes also need shared attachment
storage, since a node's local cache directory cannot serve files uploaded via another node.
Clustering is enabled by setting `experimental-cluster-listen`. Nodes find each other through the shared database and
forward published messages to each other over a private HTTP listener. Every node also reads new messages from the
database and delivers those its subscribers have not had yet, so a message a peer failed to forward still arrives, a
little later. Scheduled (delayed) messages are claimed by one node, so each is sent once, and jobs that must run only
once (pruning, web push expiry warnings, the published-messages total) run on the cluster leader only.
Example config for a node (all nodes are identical except `experimental-cluster-node-id`, `experimental-cluster-listen`
and `experimental-cluster-advertise-url`):
``` yaml
base-url: "https://ntfy.example.com"
database-url: "postgres://user:pass@dbhost:5432/ntfy"
experimental-cluster-node-id: "node-1" # stable identity, e.g. the hostname
experimental-cluster-listen: "10.0.0.5:2587" # private network interface
experimental-cluster-advertise-url: "http://10.0.0.5:2587" # how the other nodes reach this one
experimental-cluster-secret: "long-random-shared-secret" # must match on all nodes
experimental-cluster-role: "active" # or "passive", see below
```
A few important notes:
* **The cluster listener must not be publicly reachable.** It accepts messages from anyone with the shared secret, so
bind it to a private network interface. Every node must reach every other node's listener: a firewall that drops the
cluster port in one direction delays all messages that way to the database read (see [cluster status](#cluster-status)).
* `experimental-cluster-node-id` must be **stable across restarts** (use the hostname).
* **Active and passive nodes:** an `active` node serves clients behind the load balancer and can become the leader. A
`passive` node never leads and never becomes ready, so it stays out of the load balancer; use it for test clients
only, since while it is cut off from the database, its clients can miss messages published elsewhere.
* **Health checks:** point your load balancer at `/v1/ready`, not `/v1/health` (see [Health checks](#health-checks)).
* **Draining on shutdown:** when an active node gets `SIGTERM` (e.g. `systemctl restart ntfy` during a deploy) and
another active node is up, it moves its clients to the other nodes gradually before it exits, so a restart does not
cause a reconnect storm. See [draining on shutdown](#draining-on-shutdown).
* **Limits are per node.** Rate limits and daily quotas are counted on each node separately, so a visitor spreading
requests across N nodes gets N times the limits. Access changes (e.g. a revoked token) reach the other nodes when
their access cache reloads.
* Messages published with `cache: no` are not in the database, so for those, delivery across nodes is best effort.
### Draining on shutdown
Restarting a node drops all of its subscriber connections at once, and all of its clients reconnect to the other nodes
in one burst. To avoid that, an active node that gets `SIGTERM` while another active node is up **drains** before it
exits, moving its clients to the other nodes gradually, and without failed requests:
1. **Withdrawing** (`experimental-cluster-drain-delay`, default 15 seconds): `/v1/ready` fails at once, so your load
balancer stops sending the node new requests, but the node still serves everything, so requests already on their
way succeed. In the cluster, the node acts as passive from now on: another node takes over the leader's jobs, and
it sends no scheduled (delayed) messages.
2. **Read-only** (`experimental-cluster-drain-duration`, default 1 minute): new subscriptions and publishes get HTTP
503 (incoming email gets a temporary SMTP error, so mail servers retry), and the open subscriber connections are
closed evenly over the duration (all at once with `0`). The clients reconnect, through the load balancer, to the
other nodes, and catch up on what they missed. Once no connections are left, the node exits.
| | Active | Passive | Draining: withdrawing | Draining: read-only |
|---|---|---|---|---|
| `/v1/ready` | 200 | 503 | 503, `"role": "passive", "draining": true` | 503, `"role": "passive", "draining": true` |
| New subscriptions (streams, WebSockets) | accepted | accepted | accepted | 503 |
| Open subscriber connections | kept | kept | kept | closed gradually |
| Polls (`poll=1`) | served | served | served | served |
| Publishes (HTTP and email) | served | served | served | rejected (503, SMTP 451) |
| Leads the cluster's jobs, sends delayed messages | may | never | never | never |
Your load balancer should stop routing to a node once its `/v1/ready` fails, and your service manager must wait for
both phases (e.g. systemd's `TimeoutStopSec`, default 90 seconds). A passive node, and the last active node, exit at
once.
### Cluster status
`/v1/health` only covers a node's own database heartbeat. A link between two nodes that works in one direction only,
e.g. because a firewall drops the cluster port one way, keeps both nodes healthy while messages published on one of them
reach the other only late, through the database. To see each node's view of its peers, run `ntfy cluster status` on the
node. It reads `experimental-cluster-listen` and `experimental-cluster-secret` from the server config, and shows per
peer the message batches sent there and received from there:
```
$ ntfy cluster status
Node nyc3-app-2: passive, not leader, healthy
PEER ROLE LIVE HEARTBEAT SENT OK FAILED QUEUED DROPPED LAST SENT RECEIVED LAST RECEIVED
nyc3-app-1 active yes 2s ago 0 219 0 0 never 5120 1s ago
Problems:
nyc3-app-1: no batch accepted, 219 failed; last error 10s ago: Post "http://10.108.0.2:2587/v1/cluster/message": context deadline exceeded
```
`ntfy cluster status --all` also fetches every live peer's view and lists each link between the nodes once, with what
the sender sent next to what the receiver received, so a one-way link shows from either node:
```
$ ntfy cluster status --all
NODE ROLE LEADER HEALTHY
nyc3-app-1 active yes yes
nyc3-app-2 passive no yes
LINK SENT OK FAILED LAST SENT RECEIVED LAST RECEIVED
nyc3-app-1 -> nyc3-app-2 5120 0 1s ago 5120 1s ago
nyc3-app-2 -> nyc3-app-1 0 219 never 0 never
Problems:
nyc3-app-2 -> nyc3-app-1: no batch accepted, 219 failed; last error 10s ago: context deadline exceeded
```
A peer that left the registry is listed as not live with when it was last seen, and is forgotten after 24 hours.
`ntfy cluster status --json` prints the same as JSON. It is served on the cluster listener at `/v1/cluster/status`,
authenticated with the cluster secret in the `X-Cluster-Secret` header. With [metrics](#monitoring) enabled, the same
counters are exported per peer: `ntfy_cluster_peer_batches_sent_total{peer,result}` (`result` is `ok` or `error`),
`ntfy_cluster_peer_batches_received_total{peer}`, and `ntfy_cluster_peer_last_send_success_timestamp_seconds{peer}`. A
peer's series start at zero when it is first seen (a last-success time of 0 means never) and are removed once it has
been gone for 24 hours. To alert on a broken link, look at failures rather than at the last-success time, since a node
with nothing to send never succeeds either:
```
increase(ntfy_cluster_peer_batches_sent_total{result="error"}[5m]) > 0
unless increase(ntfy_cluster_peer_batches_sent_total{result="ok"}[5m]) > 0
```
## Health checks
A preliminary health check API endpoint is exposed at `/v1/health`. The endpoint returns a `json` response in the format shown below.
If a non-200 HTTP status code is returned or if the returned `healthy` field is `false` the ntfy service should be considered as unhealthy.
```json
{"healthy":true}
{"healthy":true,"role":"active"}
```
Without clustering, the server is healthy whenever it answers; the endpoint does not check the database. With clustering
(experimental, see [Clustering](#clustering-experimental)), it is unhealthy while its cluster heartbeat, a database write every
few seconds, has failed for 30 seconds.
For load balancers, use `/v1/ready` instead: it returns 200 only if the server should receive clients. Without
clustering that is always the case. With clustering, a passive node is never ready, and an unhealthy active node is not ready
while another active node can take its clients. The only active node stays ready even when it is unhealthy, since there is
nowhere better for its clients. A node [draining](#draining-on-shutdown) before a shutdown is not ready either: it acts
as passive, and says so:
```json
{"ready":true,"role":"active"}
{"ready":false,"role":"passive","draining":true}
```
See [Installation for Docker](install.md#docker) for an example of how this could be used in a `docker-compose` environment.
@@ -2340,7 +2504,7 @@ variable before running the `ntfy` command (e.g. `export NTFY_LISTEN_HTTP=:80`).
| Config option | Env variable | Format | Default | Description |
|--------------------------------------------|-------------------------------------------------|-----------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `base-url` | `NTFY_BASE_URL` | *URL* | - | Public facing base URL of the service (e.g. `https://ntfy.sh`) |
| `listen-http` | `NTFY_LISTEN_HTTP` | `[host]:port` | `:80` | Listen address for the HTTP web server |
| `listen-http` | `NTFY_LISTEN_HTTP` | `[host]:port` | `:80` | Listen address for the HTTP web server; several comma-separated addresses are allowed |
| `listen-https` | `NTFY_LISTEN_HTTPS` | `[host]:port` | - | Listen address for the HTTPS web server. If set, you also need to set `key-file` and `cert-file`. |
| `listen-unix` | `NTFY_LISTEN_UNIX` | *filename* | - | Path to a Unix socket to listen on |
| `listen-unix-mode` | `NTFY_LISTEN_UNIX_MODE` | *file mode* | *system default* | File mode of the Unix socket, e.g. 0700 or 0777 |
@@ -2348,6 +2512,14 @@ variable before running the `ntfy` command (e.g. `export NTFY_LISTEN_HTTP=:80`).
| `cert-file` | `NTFY_CERT_FILE` | *filename* | - | HTTPS/TLS certificate file, only used if `listen-https` is set. |
| `firebase-key-file` | `NTFY_FIREBASE_KEY_FILE` | *filename* | - | If set, also publish messages to a Firebase Cloud Messaging (FCM) topic for your app. This is optional and only required to save battery when using the Android app. See [Firebase (FCM)](#firebase-fcm). |
| `database-url` | `NTFY_DATABASE_URL` | *string (connection URL)* | - | PostgreSQL connection string (e.g. `postgres://user:pass@host:5432/ntfy`). If set, uses PostgreSQL for all database-backed stores (message cache, user manager, web push) instead of SQLite. See [database options](#database-options). |
| `experimental-cluster-node-id` | `NTFY_EXPERIMENTAL_CLUSTER_NODE_ID` | *string* | - | Experimental, not ready for production. Stable per-node identifier (e.g. the hostname). |
| `experimental-cluster-listen` | `NTFY_EXPERIMENTAL_CLUSTER_LISTEN` | `[host]:port` | - | Experimental, not ready for production. Private listen address for node-to-node traffic; setting it enables clustering (requires `database-url`). |
| `experimental-cluster-advertise-url` | `NTFY_EXPERIMENTAL_CLUSTER_ADVERTISE_URL` | *URL* | `http://<experimental-cluster-listen>` | Experimental, not ready for production. URL under which other nodes reach this node's cluster listener. |
| `experimental-cluster-secret` | `NTFY_EXPERIMENTAL_CLUSTER_SECRET` | *string* | - | Experimental, not ready for production. Shared secret authenticating node-to-node traffic; must match on all nodes. |
| `experimental-cluster-batch-linger` | `NTFY_EXPERIMENTAL_CLUSTER_BATCH_LINGER` | *duration* | `500ms` | Experimental, not ready for production. How long messages wait to form a node-to-node delivery batch; `0` sends immediately. |
| `experimental-cluster-role` | `NTFY_EXPERIMENTAL_CLUSTER_ROLE` | `active` or `passive` | `active` | Experimental, not ready for production. A `passive` node never leads the cluster's singleton jobs and never closes its subscribers to send them to another node; other active nodes do not count it as a place for theirs. Use it for a node that is not behind the load balancer (check `/v1/ready`, see [Health checks](#health-checks)). While a passive node is cut off from the database, its clients can miss messages published on other nodes, so give it test clients only. |
| `experimental-cluster-drain-delay` | `NTFY_EXPERIMENTAL_CLUSTER_DRAIN_DELAY` | *duration* | `15s` | Experimental, not ready for production. When [draining on shutdown](#draining-on-shutdown), how long a node withdraws: it is not ready but still serves everything, so the load balancer stops sending it requests before the node turns read-only. |
| `experimental-cluster-drain-duration` | `NTFY_EXPERIMENTAL_CLUSTER_DRAIN_DURATION` | *duration* | `1m` | Experimental, not ready for production. When [draining on shutdown](#draining-on-shutdown), how long an active node takes to close its subscriber connections, so they reconnect to the other nodes gradually; `0` closes them all at once. The service manager's stop timeout must be longer than the delay plus this. |
| `database-replica-urls` | `NTFY_DATABASE_REPLICA_URLS` | *list of strings (connection URLs)* | - | PostgreSQL read replica connection strings. Non-critical read-only queries are distributed across replicas (round-robin) with automatic fallback to primary. Requires `database-url`. |
| `cache-file` | `NTFY_CACHE_FILE` | *filename* | - | If set, messages are cached in a local SQLite database instead of only in-memory. This allows for service restarts without losing messages in support of the since= parameter. See [message cache](#message-cache). |
| `cache-duration` | `NTFY_CACHE_DURATION` | *duration* | 12h | Duration for which messages will be buffered before they are deleted. This is required to support the `since=...` and `poll=1` parameter. Set this to `0` to disable the cache entirely. |
@@ -2363,7 +2535,7 @@ variable before running the `ntfy` command (e.g. `export NTFY_LISTEN_HTTP=:80`).
| `attachment-cache-dir` | `NTFY_ATTACHMENT_CACHE_DIR` | *directory or S3 URL* | - | Cache directory for attached files, or S3 URL for object storage (format: `s3://KEY:SECRET@BUCKET[/PREFIX]?region=REGION[&endpoint=ENDPOINT][&disable_http2=true]`). |
| `attachment-total-size-limit` | `NTFY_ATTACHMENT_TOTAL_SIZE_LIMIT` | *size* | 5G | Limit of the on-disk attachment cache directory. If the limits is exceeded, new attachments will be rejected. |
| `attachment-file-size-limit` | `NTFY_ATTACHMENT_FILE_SIZE_LIMIT` | *size* | 15M | Per-file attachment size limit (e.g. 300k, 2M, 100M). Larger attachment will be rejected. |
| `attachment-expiry-duration` | `NTFY_ATTACHMENT_EXPIRY_DURATION` | *duration* | 3h | Duration after which uploaded attachments will be deleted (e.g. 3h, 20h). Strongly affects `visitor-attachment-total-size-limit`. |
| `attachment-expiry-duration` | `NTFY_ATTACHMENT_EXPIRY_DURATION` | *duration* | 3h | Duration after which uploaded attachments will be deleted (e.g. 3h, 20h). Capped at `cache-duration`. Strongly affects `visitor-attachment-total-size-limit`. |
| `smtp-sender-addr` | `NTFY_SMTP_SENDER_ADDR` | `host:port` | - | SMTP server address to allow email sending |
| `smtp-sender-user` | `NTFY_SMTP_SENDER_USER` | *string* | - | SMTP user; only used if e-mail sending is enabled |
| `smtp-sender-pass` | `NTFY_SMTP_SENDER_PASS` | *string* | - | SMTP password; only used if e-mail sending is enabled |
@@ -2454,7 +2626,7 @@ OPTIONS:
--log-file value, --log_file value set log file, default is STDOUT [$NTFY_LOG_FILE]
--config value, -c value config file (default: "/etc/ntfy/server.yml") [$NTFY_CONFIG_FILE]
--base-url value, --base_url value, -B value externally visible base URL for this host (e.g. https://ntfy.sh) [$NTFY_BASE_URL]
--listen-http value, --listen_http value, -l value ip:port used as HTTP listen address (default: ":80") [$NTFY_LISTEN_HTTP]
--listen-http value, --listen_http value, -l value ip:port used as HTTP listen address; several comma-separated addresses are allowed (default: ":80") [$NTFY_LISTEN_HTTP]
--listen-https value, --listen_https value, -L value ip:port used as HTTPS listen address [$NTFY_LISTEN_HTTPS]
--listen-unix value, --listen_unix value, -U value listen on unix socket path [$NTFY_LISTEN_UNIX]
--listen-unix-mode value, --listen_unix_mode value file permissions of unix socket, e.g. 0700 (default: system default) [$NTFY_LISTEN_UNIX_MODE]
docs/contributing.md
+2
View File
@@ -13,6 +13,8 @@ If you'd like to contribute code to ntfy:
before investing significant time
4. Submit a pull request on GitHub
If you use AI tools to write your code, please read the [AI policy](ai-policy.md) first.
All contributions are welcome, from small bug fixes to major features.
## Translations
docs/develop.md
+61
View File
@@ -279,6 +279,67 @@ Reference: <https://stackoverflow.com/questions/34160509/options-for-testing-ser
2. Run the server (step 2 above)
3. Open <http://localhost/>
### Run the web app end-to-end tests
The web app has [Playwright](https://playwright.dev/) end-to-end tests in `web/e2e/`. They drive a real (headless) Chromium
against a real ntfy server, and cover subscribing, publishing, notifications, signup/login, account settings, access tokens,
reservations and languages. To build the web app and the server, and run all tests against a fresh local server, type:
``` shell
$ make web-e2e
...
56 passed
```
This builds the binary `build/ntfy-e2e` (with the web app embedded), downloads Chromium if needed, and starts the server
on `127.0.0.1:12586` with a throwaway database in `build/e2e/` (see `web/e2e/server.mjs`). The server is stopped when the
tests are done.
After `make web-e2e` has run once, you can run the tests directly from `web/`, e.g. a single spec file, or with a visible
browser. The binary is not rebuilt this way, so **run `make web-e2e` again after changing the web app or the server**:
``` shell
$ cd web
$ npx playwright test settings.spec.js # Only the tests in one file
$ npx playwright test -g "access token" # Only tests whose name matches
$ npx playwright test --headed # Watch the browser while the tests run
$ npx playwright test --ui # Interactive UI: pick tests, step through them
```
If a test fails, its trace is kept in `web/test-results/`. Open it with `npx playwright show-trace <path>/trace.zip` to see
every step with screenshots, the DOM, console logs and network requests. To get traces and an HTML report for **all**
tests, including the passing ones:
``` shell
$ npx playwright test --trace on --reporter=list,html
$ npx playwright show-report
```
#### Against an existing server
The tests can also run against an existing server (e.g. staging) instead of a local one. They create (and delete) their own
users and topics on that server, so **don't run them against production**. You'll need:
- an admin access token (e.g. `ntfy token add <admin-user>`), which is used to create and delete test users
- an existing tier that allows reservations (e.g. `ntfy tier add --reservation-limit 10 e2e`), which is assigned to the
test users that need a paid account
``` shell
$ cd web
$ NTFY_E2E_BASE_URL=https://staging.example.com \
NTFY_E2E_ADMIN_TOKEN=tk_... \
NTFY_E2E_TIER=e2e \
npx playwright test --workers 2
```
Tests for features the server has disabled (e.g. signup or reservations, as reported by `/v1/config`) are skipped. A few
things to keep in mind:
- Every run signs up one account through the web app, and ntfy only allows a few signups per IP per day, so you can run
the full suite about 6 times per day.
- All tests come from your IP, so request limits may kick in; use fewer workers (e.g. `--workers 2`) if you see `429`s.
- The reservation tests expect anonymous users to be allowed to publish and subscribe to topics that aren't reserved
(`auth-default-access: read-write`).
### Build the docs
The sources for the docs live in `docs/`. Similarly to the web app, you can simply run `make docs` to build the
documentation. As long as you have `mkdocs` installed (see above), this should work fine:
docs/faq.md
+2 -1
View File
@@ -87,7 +87,8 @@ In the default configuration, you'll be able to do 60 requests as a burst, and t
choose a random 10 digit topic name using only A-Z, a-z, 0-9, _ and -, there are 64^10 possible topic names. Even if you
could do hundreds of requests per seconds (which you cannot), it would take many years to brute force a topic name.
For ntfy.sh, there's even a fail2ban in place which will ban your IP pretty quickly.
For ntfy.sh, there are additional protections in place which will ban your IP pretty quickly. If you think you have
been banned from ntfy.sh, you can check at [am-i-banned.ntfy.sh](https://am-i-banned.ntfy.sh/).
## Where can I donate?
I have just very recently started accepting donations via [GitHub Sponsors](https://github.com/sponsors/binwiederhier).
docs/install.md
+38 -38
View File
@@ -34,37 +34,37 @@ as a service starting at boot time.
=== "x86_64/amd64"
```bash
wget https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_amd64.tar.gz
tar zxvf ntfy_2.28.0_linux_amd64.tar.gz
sudo cp -a ntfy_2.28.0_linux_amd64/ntfy /usr/local/bin/ntfy
sudo mkdir /etc/ntfy && sudo cp ntfy_2.28.0_linux_amd64/{client,server}/*.yml /etc/ntfy
wget https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_amd64.tar.gz
tar zxvf ntfy_2.35.0_linux_amd64.tar.gz
sudo cp -a ntfy_2.35.0_linux_amd64/ntfy /usr/local/bin/ntfy
sudo mkdir /etc/ntfy && sudo cp ntfy_2.35.0_linux_amd64/{client,server}/*.yml /etc/ntfy
sudo ntfy serve
```
=== "armv6"
```bash
wget https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_armv6.tar.gz
tar zxvf ntfy_2.28.0_linux_armv6.tar.gz
sudo cp -a ntfy_2.28.0_linux_armv6/ntfy /usr/bin/ntfy
sudo mkdir /etc/ntfy && sudo cp ntfy_2.28.0_linux_armv6/{client,server}/*.yml /etc/ntfy
wget https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_armv6.tar.gz
tar zxvf ntfy_2.35.0_linux_armv6.tar.gz
sudo cp -a ntfy_2.35.0_linux_armv6/ntfy /usr/bin/ntfy
sudo mkdir /etc/ntfy && sudo cp ntfy_2.35.0_linux_armv6/{client,server}/*.yml /etc/ntfy
sudo ntfy serve
```
=== "armv7/armhf"
```bash
wget https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_armv7.tar.gz
tar zxvf ntfy_2.28.0_linux_armv7.tar.gz
sudo cp -a ntfy_2.28.0_linux_armv7/ntfy /usr/bin/ntfy
sudo mkdir /etc/ntfy && sudo cp ntfy_2.28.0_linux_armv7/{client,server}/*.yml /etc/ntfy
wget https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_armv7.tar.gz
tar zxvf ntfy_2.35.0_linux_armv7.tar.gz
sudo cp -a ntfy_2.35.0_linux_armv7/ntfy /usr/bin/ntfy
sudo mkdir /etc/ntfy && sudo cp ntfy_2.35.0_linux_armv7/{client,server}/*.yml /etc/ntfy
sudo ntfy serve
```
=== "arm64"
```bash
wget https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_arm64.tar.gz
tar zxvf ntfy_2.28.0_linux_arm64.tar.gz
sudo cp -a ntfy_2.28.0_linux_arm64/ntfy /usr/bin/ntfy
sudo mkdir /etc/ntfy && sudo cp ntfy_2.28.0_linux_arm64/{client,server}/*.yml /etc/ntfy
wget https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_arm64.tar.gz
tar zxvf ntfy_2.35.0_linux_arm64.tar.gz
sudo cp -a ntfy_2.35.0_linux_arm64/ntfy /usr/bin/ntfy
sudo mkdir /etc/ntfy && sudo cp ntfy_2.35.0_linux_arm64/{client,server}/*.yml /etc/ntfy
sudo ntfy serve
```
@@ -84,25 +84,25 @@ Install the ntfy server unit file (which contains parameters to start the servic
=== "x86_64/amd64"
```bash
sudo mv ntfy_2.28.0_linux_amd64/server/ntfy.service /etc/systemd/system/
sudo mv ntfy_2.35.0_linux_amd64/server/ntfy.service /etc/systemd/system/
sudo chmod 644 /etc/systemd/system/ntfy.service
```
=== "armv6"
```bash
sudo mv ntfy_2.28.0_linux_armv6/server/ntfy.service /etc/systemd/system/
sudo mv ntfy_2.35.0_linux_armv6/server/ntfy.service /etc/systemd/system/
sudo chmod 644 /etc/systemd/system/ntfy.service
```
=== "armv7/armhf"
```bash
sudo mv ntfy_2.28.0_linux_armv7/server/ntfy.service /etc/systemd/system/
sudo mv ntfy_2.35.0_linux_armv7/server/ntfy.service /etc/systemd/system/
sudo chmod 644 /etc/systemd/system/ntfy.service
```
=== "arm64"
```bash
sudo mv ntfy_2.28.0_linux_arm64/server/ntfy.service /etc/systemd/system/
sudo mv ntfy_2.35.0_linux_arm64/server/ntfy.service /etc/systemd/system/
sudo chmod 644 /etc/systemd/system/ntfy.service
```
@@ -118,25 +118,25 @@ Install the ntfy server service script:
=== "x86_64/amd64"
```bash
sudo mv ntfy_2.28.0_linux_amd64/server/ntfy.openrc /etc/init.d/ntfy
sudo mv ntfy_2.35.0_linux_amd64/server/ntfy.openrc /etc/init.d/ntfy
sudo chmod 755 /etc/init.d/ntfy
```
=== "armv6"
```bash
sudo mv ntfy_2.28.0_linux_armv6/server/ntfy.openrc /etc/init.d/ntfy
sudo mv ntfy_2.35.0_linux_armv6/server/ntfy.openrc /etc/init.d/ntfy
sudo chmod 755 /etc/init.d/ntfy
```
=== "armv7/armhf"
```bash
sudo mv ntfy_2.28.0_linux_armv7/server/ntfy.openrc /etc/init.d/ntfy
sudo mv ntfy_2.35.0_linux_armv7/server/ntfy.openrc /etc/init.d/ntfy
sudo chmod 755 /etc/init.d/ntfy
```
=== "arm64"
```bash
sudo mv ntfy_2.28.0_linux_arm64/server/ntfy.openrc /etc/init.d/ntfy
sudo mv ntfy_2.35.0_linux_arm64/server/ntfy.openrc /etc/init.d/ntfy
sudo chmod 755 /etc/init.d/ntfy
```
@@ -204,7 +204,7 @@ Manually installing the .deb file:
=== "x86_64/amd64"
```bash
wget https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_amd64.deb
wget https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_amd64.deb
sudo dpkg -i ntfy_*.deb
sudo systemctl enable ntfy
sudo systemctl start ntfy
@@ -212,7 +212,7 @@ Manually installing the .deb file:
=== "armv6"
```bash
wget https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_armv6.deb
wget https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_armv6.deb
sudo dpkg -i ntfy_*.deb
sudo systemctl enable ntfy
sudo systemctl start ntfy
@@ -220,7 +220,7 @@ Manually installing the .deb file:
=== "armv7/armhf"
```bash
wget https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_armv7.deb
wget https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_armv7.deb
sudo dpkg -i ntfy_*.deb
sudo systemctl enable ntfy
sudo systemctl start ntfy
@@ -228,7 +228,7 @@ Manually installing the .deb file:
=== "arm64"
```bash
wget https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_arm64.deb
wget https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_arm64.deb
sudo dpkg -i ntfy_*.deb
sudo systemctl enable ntfy
sudo systemctl start ntfy
@@ -238,28 +238,28 @@ Manually installing the .deb file:
=== "x86_64/amd64"
```bash
sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_amd64.rpm
sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_amd64.rpm
sudo systemctl enable ntfy
sudo systemctl start ntfy
```
=== "armv6"
```bash
sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_armv6.rpm
sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_armv6.rpm
sudo systemctl enable ntfy
sudo systemctl start ntfy
```
=== "armv7/armhf"
```bash
sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_armv7.rpm
sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_armv7.rpm
sudo systemctl enable ntfy
sudo systemctl start ntfy
```
=== "arm64"
```bash
sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_linux_arm64.rpm
sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_linux_arm64.rpm
sudo systemctl enable ntfy
sudo systemctl start ntfy
```
@@ -301,18 +301,18 @@ pkg install go-ntfy
## macOS
The [ntfy CLI](subscribe/cli.md) (`ntfy publish` and `ntfy subscribe` only) is supported on macOS as well.
To install, please [download the tarball](https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_darwin_all.tar.gz),
To install, please [download the tarball](https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_darwin_all.tar.gz),
extract it and place it somewhere in your `PATH` (e.g. `/usr/local/bin/ntfy`).
If run as `root`, ntfy will look for its config at `/etc/ntfy/client.yml`. For all other users, it'll look for it at
`~/Library/Application Support/ntfy/client.yml` (sample included in the tarball).
```bash
curl -L https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_darwin_all.tar.gz > ntfy_2.28.0_darwin_all.tar.gz
tar zxvf ntfy_2.28.0_darwin_all.tar.gz
sudo cp -a ntfy_2.28.0_darwin_all/ntfy /usr/local/bin/ntfy
curl -L https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_darwin_all.tar.gz > ntfy_2.35.0_darwin_all.tar.gz
tar zxvf ntfy_2.35.0_darwin_all.tar.gz
sudo cp -a ntfy_2.35.0_darwin_all/ntfy /usr/local/bin/ntfy
mkdir ~/Library/Application\ Support/ntfy
cp ntfy_2.28.0_darwin_all/client/client.yml ~/Library/Application\ Support/ntfy/client.yml
cp ntfy_2.35.0_darwin_all/client/client.yml ~/Library/Application\ Support/ntfy/client.yml
ntfy --help
```
@@ -333,7 +333,7 @@ brew install ntfy
The ntfy server and CLI are fully supported on Windows. You can run the ntfy server directly or as a Windows service.
To install, you can either
* [Download the latest ZIP](https://github.com/binwiederhier/ntfy/releases/download/v2.28.0/ntfy_2.28.0_windows_amd64.zip),
* [Download the latest ZIP](https://github.com/binwiederhier/ntfy/releases/download/v2.35.0/ntfy_2.35.0_windows_amd64.zip),
extract it and place the `ntfy.exe` binary somewhere in your `%Path%`.
* Or install ntfy from the [Scoop](https://scoop.sh) main repository via `scoop install ntfy`
docs/integrations.md
+14 -1
View File
@@ -44,6 +44,7 @@ I've added a ⭐ to projects or posts that have a significant following, or had
- [Beszel](https://beszel.dev/guide/notifications/ntfy) - Server monitoring platform
- [Simple Observability](https://simpleobservability.com/docs/alerts/ntfy) - Server monitoring and observability platform
- [Sifio](https://sifio.net) - Aggregate updates from RSS, social media, and other sources, then deliver them to ntfy, Slack, Notion and more
- [SolidPing](https://docs.solidping.io/docs/configuration/notifications#ntfy) - Uptime and synthetic monitoring
## Integration via HTTP/SMTP/etc.
@@ -88,11 +89,12 @@ I've added a ⭐ to projects or posts that have a significant following, or had
- [ntfy-logging](https://github.com/Pimak/ntfy-logging) - Turns JVM error logs into ntfy notifications, with zero-code adapters for java.util.logging, Logback, Log4j2, Spring Boot, Micronaut and Quarkus (Java)
## CLIs + GUIs
- [ntfy-wear](https://github.com/cloudsliberty/ntfy-wear) - Lightweight ntfy client for Wear OS round-screen smartwatches.
- [ntfy.sh.sh](https://github.com/mininmobile/ntfy.sh.sh) - Run scripts on ntfy.sh events
- [ntfy-desktop](https://codeberg.org/zvava/ntfy-desktop) - Cross-platform desktop application for ntfy
- [ntfy-desktop](https://github.com/Aetherinox/ntfy-desktop) - Desktop client for Windows, Linux, and MacOS with push notifications
- [ntfy Desktop (Windows)](https://github.com/simoneferrari/ntfy-desktop) - Native Windows desktop client with multi-server support, toast notifications and message history, built with WPF and .NET (C#)
- [Notidesk](https://github.com/onatakduman/ntfy-desktop-windows) - Native Windows desktop client built with WinUI 3, with toast notifications, tray support and publishing; available on the [Microsoft Store](https://apps.microsoft.com/detail/9PJDD0JKR719) (C#)
- [wio-ntfy-ticker](https://github.com/nachotp/wio-ntfy-ticker) - Ticker display for a ntfy.sh topic
- [ntfysh-windows](https://github.com/mshafer1/ntfysh-windows) - A ntfy client for Windows Desktop
- [ntfyr](https://github.com/haxwithaxe/ntfyr) - A simple commandline tool to send notifications to ntfy
@@ -103,6 +105,8 @@ I've added a ⭐ to projects or posts that have a significant following, or had
- [ntfyexec](https://github.com/alecthomas/ntfyexec) - Send a notification through ntfy.sh if a command fails
- [Ntfy Desktop](https://github.com/emmaexe/ntfyDesktop) - Fully featured desktop client for Linux, built with Qt and C++.
- [Ntfy App](https://github.com/rubix-studios-pty-ltd/ntfy-app) - Tauri/Rust desktop client for Windows, Linux and MacOS with push notifications.
- [mac_ntfy](https://github.com/wangty163/mac_ntfy) - Native macOS client for ntfy (Swift)
- [NtfyBar](https://github.com/roypadina/NtfyBar) - Native macOS menu-bar client: per-topic popups that stay, notifications, clipboard or shell-command actions, rules, history and sandboxed JavaScript plugins (Swift)
## Projects + scripts
@@ -153,6 +157,7 @@ I've added a ⭐ to projects or posts that have a significant following, or had
- [nlog-ntfy](https://github.com/MichelMichels/nlog-ntfy) - Send NLog messages over ntfy (C# / .NET / NLog)
- [helm-charts](https://github.com/sarab97/helm-charts) - Helm charts of some of the selfhosted services, incl. ntfy
- [ntfy_ansible_role](https://github.com/stevenengland/ntfy_ansible_role) (on [Ansible Galaxy](https://galaxy.ansible.com/stevenengland/ntfy)) - Ansible role to install ntfy
- [MASH playbook](https://github.com/mother-of-all-self-hosting/mash-playbook/blob/main/docs/services/ntfy.md) - Ansible playbook to install ntfy (and 250+ other self-hosted services) as Docker containers
- [easy2ntfy](https://github.com/chromoxdor/easy2ntfy) - Gateway for ESPeasy to receive commands through ntfy and using easyfetch (HTML/JS)
- [ntfy_lite](https://github.com/MPI-IS/ntfy_lite) - Minimalist python API for pushing ntfy notifications (Python)
- [notify](https://github.com/guanguans/notify) - 推送通知 (PHP)
@@ -193,6 +198,13 @@ I've added a ⭐ to projects or posts that have a significant following, or had
- [SIA-Server](https://github.com/ZebMcKayhan/SIA-Server) - A light weight, self-hosted notification Server for Honywell Galaxy Flex alarm systems (Python)
- [zabbix-ntfy](https://github.com/torgrimt/zabbix-ntfy) - Zabbix server Mediatype to add support for ntfy.sh services
- [Rubix Notify](https://wordpress.org/plugins/rubix-notify) - WordPress Integration with ntfy (PHP + React).
- [NotiFerry](https://notiferry.com/) - macOS menu bar app that forwards Notification Center alerts from other Mac apps to an ntfy topic on ntfy.sh or a self-hosted server (commercial, Swift)
- [Flock](https://github.com/HNF-FRN/flock) - Runs several Claude Desktop accounts side by side on Windows; its phone app starts and continues Claude Code sessions on the PC through end-to-end encrypted ntfy messages (JS)
- [epokio](https://github.com/8rulerstar/epokio) - Alerts when an ML training run stalls, hits NaN or finishes, by reading its existing logs (Python)
- [Pethost](https://pethost.dev/templates/ntfy/) - One-click ntfy server on Pethost, with an admin user made at the first start and anonymous access denied
- [Sanctumd](https://github.com/Phioster/sanctum_daemon) - Android app for Jellyfin and the *arr stack; subscribes to ntfy topics for live pushes with images, priorities and action buttons (Kotlin)
- [MeetAlert](https://github.com/roypadina/MeetAlert) - macOS meeting alerts from your calendars; when you're away from the Mac it escalates to your phone via ntfy with Ack/Join/Snooze actions (Swift)
- [aibot-hostkit](https://github.com/monarchsofchaos004-gif/aibot-hostkit) - Keeps an AI coding agent (Claude Code, Codex, Gemini CLI) running on a cloud Linux box and pushes an ntfy alert to your phone whenever it restarts (Bash)
## Blog + forum posts
@@ -321,6 +333,7 @@ ntfy community. Thanks to everyone running a public server. **You guys rock!**
| [ntfy.mzte.de](https://ntfy.mzte.de/) | 🇩🇪 Germany |
| [ntfy.hostux.net](https://ntfy.hostux.net/) | 🇫🇷 France |
| [ntfy.fossman.de](https://ntfy.fossman.de/) | 🇩🇪 Germany |
| [notify.utilibre.org](https://notify.utilibre.org/) | 🇩🇪 Germany |
Please be aware that **server operators can log your messages**. The project also cannot guarantee the reliability
and uptime of third party servers, so use of each server is **at your own discretion**.
docs/publish.md
+32 -4
View File
@@ -666,7 +666,7 @@ them with a comma, e.g. `tag1,tag2,tag3`.
_Supported on:_ :material-android: :material-firefox:
You can format messages using [Markdown](https://www.markdownguide.org/basic-syntax/) 🤩. That means you can use
**bold text**, *italicized text*, links, images, and more. Supported Markdown features (web app only for now):
**bold text**, *italicized text*, links, images, and more. Supported Markdown features:
- [Emphasis](https://www.markdownguide.org/basic-syntax/#emphasis) such as **bold** (`**bold**`), *italics* (`*italics*`)
- [Links](https://www.markdownguide.org/basic-syntax/#links) (`[some tool](https://ntfy.sh)`)
@@ -2591,9 +2591,22 @@ You can delay the delivery of messages and let ntfy send them at a later date. T
reminders or even to execute commands at a later date (if your subscriber acts on messages).
Usage is pretty straight forward. You can set the delivery time using the `X-Delay` header (or any of its aliases: `Delay`,
`X-At`, `At`, `X-In` or `In`), either by specifying a Unix timestamp (e.g. `1639194738`), a duration (e.g. `30m`,
`3h`, `2 days`), or a natural language time string (e.g. `10am`, `8:30pm`, `tomorrow, 3pm`, `Tuesday, 7am`,
[and more](https://github.com/olebedev/when)).
`X-At`, `At`, `X-In` or `In`), either by specifying a Unix timestamp (e.g. `1639194738`), an RFC3339/ISO-8601 absolute
timestamp (e.g. `2026-09-02T10:00:00Z`), a duration (e.g. `30m`, `3h`, `2 days`), or a natural language time string
(e.g. `10am`, `8:30pm`, `tomorrow, 3pm`, `Tuesday, 7am`, [and more](https://github.com/olebedev/when)).
To interpret a natural language time in a specific timezone, set the `X-Timezone` header
(alias: `Timezone`), the `timezone` query parameter, or the `timezone` JSON field to an
IANA timezone name such as `Asia/Tokyo` or `America/New_York`:
```
curl -H "At: tomorrow, 10am" -H "Timezone: Asia/Tokyo" -d "Good morning" ntfy.sh/hello
```
Without a timezone, the server's local timezone is used as before. The web app sends your
browser's timezone automatically when scheduling a message. Relative durations such as
`1h` and Unix timestamps still refer to the same instant regardless of timezone. Invalid
timezone names return HTTP 400 when a delay is supplied; without a delay, the timezone is ignored.
As of today, the minimum delay you can set is **10 seconds** and the maximum delay is **3 days**. This can be configured
with the `message-delay-limit` option.
@@ -2607,6 +2620,7 @@ to be delivered in 3 days, it'll remain in the cache for 3 days and 12 hours. Al
```
curl -H "At: tomorrow, 10am" -d "Good morning" ntfy.sh/hello
curl -H "In: 30min" -d "It's 30 minutes later now" ntfy.sh/reminder
curl -H "Delay: 2026-09-02T10:00:00Z" -d "RFC3339 timestamps are awesome" ntfy.sh/itsarfc3339system
curl -H "Delay: 1639194738" -d "Unix timestamps are awesome" ntfy.sh/itsaunixsystem
```
@@ -2685,6 +2699,7 @@ Here are a few examples (assuming today's date is **12/10/2021, 9am, Eastern Tim
<tr><td><code>1 day</code></td><td>12/<b>11</b>/2021, 9am</td><td>24 hours from now</td></tr>
<tr><td><code>10am</code></td><td>12/10/2021, <b>10am</b></td><td>Today at 10am (same day, because it's only 9am)</td></tr>
<tr><td><code>8am</code></td><td>12/<b>11</b>/2021, <b>8am</b></td><td>Tomorrow at 8am (because it's 9am already)</td></tr>
<tr><td><code>2026-12-10T11:00:00-05:00</code></td><td>12/10/2026, <b>11am</b> (EST)</td><td>Absolute RFC3339 timestamp with timezone offset</td></tr>
<tr><td><code>1639152000</code></td><td>12/10/2021, 11am (EST)</td><td> Today at 11am (EST)</td></tr>
</tbody></table>
</td>
@@ -3719,6 +3734,7 @@ all the supported fields:
| `icon` | - | *string* | `https://example.com/icon.png` | URL to use as notification [icon](#icons) |
| `filename` | - | *string* | `file.jpg` | File name of the attachment |
| `delay` | - | *string* | `30min`, `9am` | Timestamp or duration for delayed delivery |
| `timezone` | - | *string* | `Asia/Tokyo` | Timezone for natural language delayed delivery times |
| `email` | - | *e-mail address or 'yes'* | `phil@example.com` or `yes` | E-mail address for e-mail notifications, or `yes` to use your primary verified address |
| `call` | - | *phone number or 'yes'* | `+1222334444` or `yes` | Phone number to use for [voice call](#phone-calls) |
| `sequence_id` | - | *string* | `my-sequence-123` | Sequence ID for [updating/deleting notifications](#updating-deleting-notifications) |
@@ -4696,6 +4712,13 @@ authParam = base64_raw(authHeader) // -> QmFzaWMgZEdWemRIVnpaWEk2Wm1GclpYQmhjM0
// If your language does not have a function to encode raw base64, simply use normal base64
// and REMOVE TRAILING "=" characters.
```
You can use following one liner to create `auth` parameter from ntfy token using powershell:
``` powershell
(Read-Host -Prompt 'Enter Ntfy token' | ForEach-Object { ([Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes('Basic ' + [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes(":$_"))))) }).TrimEnd('=')
# If you have base64 module installed
(Read-Host -Prompt "Enter Ntfy token" | ForEach-Object { ":$_" } | ConvertTo-Base64 | ForEach-Object { "Basic $_" } | ConvertTo-Base64 ).TrimEnd('=')
```
The following command will generate the appropriate value for you on *nix systems:
@@ -4938,6 +4961,10 @@ but just in case, let's list them all:
These limits can be changed on a per-user basis using [tiers](config.md#tiers). If [payments](config.md#payments) are enabled, a user tier can be changed by purchasing
a higher tier. ntfy.sh offers multiple paid tiers, which allows for much hier limits than the ones listed above.
!!! tip
If you repeatedly exceed the limits on ntfy.sh, or otherwise abuse the service, your IP address may be temporarily
banned. You can check if you have been banned at [am-i-banned.ntfy.sh](https://am-i-banned.ntfy.sh/).
## List of all parameters
The following is a list of all parameters that can be passed when publishing a message. Parameter names are **case-insensitive**
when used in **HTTP headers**, and must be **lowercase** when used as **query parameters in the URL**. They are listed in the
@@ -4957,6 +4984,7 @@ table in their canonical form.
| `X-Priority` | `Priority`, `prio`, `p` | [Message priority](#message-priority) |
| `X-Tags` | `Tags`, `Tag`, `ta` | [Tags and emojis](#tags-emojis) |
| `X-Delay` | `Delay`, `X-At`, `At`, `X-In`, `In` | Timestamp or duration for [delayed delivery](#scheduled-delivery) |
| `X-Timezone` | `Timezone` | IANA timezone for [delayed delivery](#scheduled-delivery) |
| `X-Actions` | `Actions`, `Action` | JSON array or short format of [user actions](#action-buttons) |
| `X-Click` | `Click` | URL to open when [notification is clicked](#click-action) |
| `X-Attach` | `Attach`, `a` | URL to send as an [attachment](#attachments), as an alternative to PUT/POST-ing an attachment |
docs/releases.md
+216 -5
View File
@@ -6,13 +6,208 @@ and the [ntfy Android app](https://github.com/binwiederhier/ntfy-android/release
| Component | Version | Release date |
|------------------|---------|---------------|
| ntfy server | v2.28.0 | Aug 27, 2026 |
| ntfy server | v2.35.0 | Oct 30, 2026 |
| ntfy Android app | v1.25.2 | July 23, 2026 |
| ntfy iOS app | v1.7.0 | May 30, 2026 |
Please check out the release notes for [upcoming releases](#not-released-yet) below.
### ntfy server v2.28.0
## ntfy server v2.35.0
Released October 30, 2026
This release keeps what a user already spent when their tier changes, and makes a tier's daily email limit a hard
cap. It also fixes intermittent 403s under concurrent requests from the same IP, and the Matrix push gateway storing
messages under a protected topic on SQLite.
**Security:**
* Reject Matrix push gateway requests whose push key does not name a valid topic. Publishing strips NUL bytes and invalid UTF-8 from the topic after authorization, so an anonymous push key like `/sec%00ret` was authorized against an unmatched topic and then stored under the protected topic `secret` (SQLite only; on PostgreSQL the request was denied; thanks to [Efe Çam](https://github.com/the-arma-of-owl) for reporting)
**Features:**
* A tier change (an upgrade, a downgrade, or a tier edited in place) applies on the user's next request and keeps what was already spent; before, it refilled the request, email and attachment bandwidth limits ([#2104](https://github.com/binwiederhier/ntfy/pull/2104), [#2105](https://github.com/binwiederhier/ntfy/pull/2105))
* A tier's daily email limit is a hard cap; before, the tier's email burst came on top of it, so a user could send more emails a day than the tier allows
**Bug fixes + maintenance:**
* Fix intermittent 403s and panics in the account handlers under concurrent requests from the same IP: a login or logout could change the user under another request, so a request with valid credentials was treated as anonymous, or the other way around. Every request now gets its own visitor ([#1051](https://github.com/binwiederhier/ntfy/issues/1051), [#2113](https://github.com/binwiederhier/ntfy/issues/2113), [#2118](https://github.com/binwiederhier/ntfy/pull/2118), thanks to [@ivanetinajero](https://github.com/ivanetinajero) for reporting)
* A tier with a daily email limit of zero sends no emails; before, it could still send the server's email burst (`visitor-email-limit-burst`)
* An attachment upload rejected at a size limit is no longer charged to the attachment bandwidth limit for the part that went over
* Rate limits live in one shared limit service, keyed by `ip:<prefix>` or `user:<id>`, instead of in each visitor, which uses less memory per visitor and takes a global lock off the request path
* Accept terabyte units (e.g. `1T`) in size settings such as attachment and tier limits, matching how sizes are displayed ([#2078](https://github.com/binwiederhier/ntfy/pull/2078), thanks to [@cangen185-sudo](https://github.com/cangen185-sudo) for the contribution)
## ntfy server v2.34.0
Released October 24, 2026
This release prepares the experimental cluster to run behind load balancers: nodes drain on shutdown instead of
dropping all of their clients at once, ntfy can listen on several addresses, and access changes reach every node. It
also fixes a revoked or deleted user keeping some of their subscriptions, on every server.
**Features:**
* Allow several comma-separated addresses in `listen-http`, so a load balancer can hold more than ~63k connections to one ntfy node
* Cluster nodes drain on shutdown: an active node withdraws (stops being ready), then turns read-only and closes its subscriber connections gradually (`experimental-cluster-drain-delay`, `experimental-cluster-drain-duration`), so its clients move to the other nodes without a reconnect storm (experimental, see [draining on shutdown](config.md#draining-on-shutdown))
**Security:**
* End all of a user's subscriptions when their access to a topic is reset or the user is deleted: only the first one was ended, so the others kept receiving the topic's messages ([#1845](https://github.com/binwiederhier/ntfy/pull/1845), thanks to [@gigioneggiando](https://github.com/gigioneggiando) for reporting)
* In an experimental cluster, tell the other nodes when access changes (e.g. a reservation or a revoked permission), so they reload their access cache and disconnect subscribers who lost access; the access cache reload also catches changes made elsewhere, e.g. by the `ntfy access` CLI, and now runs every 67 seconds instead of 87
**Bug fixes + maintenance:**
* Fix a deadlock when ending subscriptions (reserving a topic with live subscribers, revoking access, deleting a user) while debug logging or `log-level-overrides` is on: the subscriptions' topic stayed locked for good
* Fix a data race when `log-level-overrides` changes (e.g. on a config reload with SIGHUP) while the server logs
## ntfy server v2.33.1
Released October 23, 2026
This is a hotfix release for the amd64 binaries of v2.33.0, which crashed with "Illegal instruction" on older CPUs.
**Bug fixes + maintenance:**
* Fix "Illegal instruction" crash of the amd64 binaries on CPUs without x86-64-v3 (AVX2/BMI2), by building releases on Ubuntu 24.04 again ([#2092](https://github.com/binwiederhier/ntfy/issues/2092), thanks to [@frank-bg](https://github.com/frank-bg) for reporting)
## ntfy server v2.33.0
Released October 23, 2026
This release builds ntfy with Go 1.27.3 (security fixes, including rejecting the compromised eMudhra intermediate
CAs), and adds monitoring for the experimental cluster: each node's view of its peers, per-peer and database
connection pool metrics.
**Features:**
* Show each cluster node's view of its peers with `ntfy cluster status`, `/v1/cluster/status` on the cluster listener, and per-peer metrics, so a link that only works in one direction is visible (experimental clustering, see [cluster status](config.md#cluster-status))
* Export PostgreSQL connection pool metrics (`ntfy_db_pool_*`: connections in use, and queries that waited for a free connection, per pool), to size `pool_max_conns` (see [PostgreSQL](config.md#postgresql))
**Security:**
* Build with Go 1.27.3, which includes the security fixes of Go 1.27.2 (`crypto/tls`, `net/http`, `html/template` and others) and 1.27.3 (`crypto/x509`: certificate chains through the compromised eMudhra intermediate CAs are rejected)
* Reject phone number verification unless Twilio confirms the code as approved, instead of accepting any successful response
* Ignore duplicate topics in a subscription and limit a subscription to 150 topics, so a single request cannot replay the same topic's cache over and over
**Bug fixes + maintenance:**
* Count published messages from every cluster node: each node adds its own publishes to the stored total instead of the leader overwriting it with its own count (experimental clustering)
## ntfy server v2.32.0
Released October 18, 2026
This release prepares ntfy.sh for a passive cluster node: jobs that write to the shared database run only on the
cluster leader, and each database store now records the oldest schema version it works with. Streaming subscriptions
also tell proxies not to buffer them.
**Features:**
* Set the `Incremental: ?1` header ([RFC 10036](https://www.rfc-editor.org/info/rfc10036/)) on streaming subscriptions, so proxies that support it forward messages without buffering ([#2056](https://github.com/binwiederhier/ntfy/issues/2056), thanks to [@bt90](https://github.com/bt90) for the suggestion)
**Bug fixes + maintenance:**
* Fix template files (`?tpl=<name>`) ignoring the priority sent with the request when the file has no `priority` key ([#2045](https://github.com/binwiederhier/ntfy/issues/2045), thanks to [@6801318d8d](https://github.com/6801318d8d) for reporting)
* Return HTTP 409 instead of 500 when deleting or changing the password of a user provisioned via `auth-users` through the admin API ([#2062](https://github.com/binwiederhier/ntfy/issues/2062), thanks to [@daniloamiranda](https://github.com/daniloamiranda) for reporting)
* Run the shared-database housekeeping (pruning expired messages, attachments and tokens, web push expiry warnings, writing the published-messages total) on the cluster leader only, so a passive node no longer overwrites the total with its own count (experimental clustering)
* Record the oldest schema version each database store still works with, so an older ntfy refuses to start on a database that a newer version migrated in a breaking way, instead of misreading it; none of today's migrations are breaking ([#2060](https://github.com/binwiederhier/ntfy/pull/2060))
## ntfy server v2.31.1
Released October 16, 2026
This release fixes the database load that made us roll back ntfy.sh from v2.31.0: reconnecting with `since=<message ID>`
read every message of the topic from the database. A new index makes these reconnects cheap, cheaper than in v2.30.0.
It also lets ntfy start with a warning on a newer database schema, so later rollbacks need no manual step.
**Bug fixes + maintenance:**
* Fix reconnects with `since=<message ID>` reading every message of the topic from the database, which overloaded ntfy.sh's database right after the v2.31.0 upgrade; the message cache gets a new index: (topic, published, id) on PostgreSQL, (topic, published, time) on SQLite ([#2058](https://github.com/binwiederhier/ntfy/pull/2058))
* Start with a warning instead of failing when the database schema is newer than this version of ntfy supports, so a rollback or a rolling cluster upgrade needs no manual schema change ([#2059](https://github.com/binwiederhier/ntfy/pull/2059))
* Web app: Fix the wrong topic opening when subscribed to the same topic on two different servers ([#858](https://github.com/binwiederhier/ntfy/issues/858), thanks to [@emigrating](https://github.com/emigrating) for reporting)
* Web app: Fix the "Upgrade to ntfy Pro" banner covering the bottom of a long topic list ([#1007](https://github.com/binwiederhier/ntfy/issues/1007), thanks to [@RuinTD](https://github.com/RuinTD) for reporting)
* Web app: The "Notifications are disabled" and "Notifications are blocked" warnings can now be closed ([#1220](https://github.com/binwiederhier/ntfy/issues/1220), thanks to [@jeremiah-k](https://github.com/jeremiah-k) for reporting)
* Web app: Fix a crash when opening the subscribe dialog before the topic list has loaded, e.g. while another tab holds the browser database ([#2014](https://github.com/binwiederhier/ntfy/issues/2014), [#1881](https://github.com/binwiederhier/ntfy/issues/1881), thanks to [@Lawrence81](https://github.com/Lawrence81) and [@controlstrip](https://github.com/controlstrip) for reporting)
**Upgrade notes:**
* This release changes the message database schema (version 16 to 17: an index for `since=<message ID>` reconnects). Downgrading to v2.31.0 or earlier afterwards requires setting the version back by hand first: `UPDATE schema_version SET version = 16 WHERE store = 'message'` on PostgreSQL, or `UPDATE schemaVersion SET version = 16 WHERE id = 1` in the SQLite `cache-file`. From this release on, a newer schema only logs a warning ([#2059](https://github.com/binwiederhier/ntfy/pull/2059))
## ntfy server v2.31.0
Released October 16, 2026
This release fixes messages that could be lost when a client reconnects, e.g. a phone switching networks, right when
messages are published: every server now reads newly written messages from its message cache once a second and sends them
to the subscribers that missed them. In load tests with server restarts and crashes, this brought lost messages down from
465 to 2 out of about 615,000. It also adds a `/v1/ready` endpoint for load balancers, and continues the experimental
clustering work with node roles.
**Features:**
* Add `/v1/ready` for load balancers, and the node's role to `/v1/health`; with clustering, a node with no other active node leads right away instead of after a one-minute hold-off ([#2050](https://github.com/binwiederhier/ntfy/pull/2050))
* Experimental clustering: add `experimental-cluster-role` to run a `passive` node that never leads and never takes over another node's subscribers, e.g. one that is not behind the load balancer ([#2049](https://github.com/binwiederhier/ntfy/pull/2049))
**Bug fixes + maintenance:**
* Fix messages lost when a client reconnects before they are written to the message cache, or when a cluster node's fan-out is lost: every node now reads newly written messages once a second and sends them to the subscribers that missed them ([#2040](https://github.com/binwiederhier/ntfy/pull/2040))
* Write user stats (messages, emails, calls) as increments, so servers sharing one database add up instead of overwriting each other's counts ([#1989](https://github.com/binwiederhier/ntfy/pull/1989))
* Don't add a visitor's anonymous usage to an account's stats when the account takes over that visitor, e.g. right after upgrading ([#2041](https://github.com/binwiederhier/ntfy/pull/2041))
**Upgrade notes:**
* A reconnect with `since=<message ID>` now also returns the messages published in the same second as that message, so none are skipped when several servers write out of order. The ntfy apps and web app drop these by ID; your own scripts may see a few duplicates ([#2040](https://github.com/binwiederhier/ntfy/pull/2040))
* Experimental clustering only: this release changes the cluster's node registry schema (version 1 to 2). Downgrading a clustered server to v2.30.0 afterwards requires setting the version back by hand first: `UPDATE schema_version SET version = 1 WHERE store = 'node_registry'` ([#2049](https://github.com/binwiederhier/ntfy/pull/2049))
## ntfy server v2.30.0
Released October 12, 2026
This release links attachments in email notifications, adds SMTP LOGIN auth (e.g. for OVH and Outlook), and fixes the
PostgreSQL memory growth that came with v2.29.0. It also ships the first experimental version of clustering: several
ntfy servers sharing one PostgreSQL database. Clustering is not ready for production yet.
**Features:**
* Link attachments in email notifications, including the file name and when the link expires ([#672](https://github.com/binwiederhier/ntfy/issues/672), thanks to [@merspieler](https://github.com/merspieler) for suggesting)
* Support SMTP servers that only offer LOGIN auth (e.g. OVH, Outlook) for sending emails ([#930](https://github.com/binwiederhier/ntfy/issues/930)/[#1530](https://github.com/binwiederhier/ntfy/pull/1530), thanks to [@wunter8](https://github.com/wunter8) for reporting and [@santanoce](https://github.com/santanoce) for the initial implementation)
* Experimental: run several ntfy servers as one cluster that shares a PostgreSQL database, enabled with `experimental-cluster-listen`; this is not ready for production yet ([#2005](https://github.com/binwiederhier/ntfy/pull/2005), [#2008](https://github.com/binwiederhier/ntfy/pull/2008), [#2012](https://github.com/binwiederhier/ntfy/pull/2012), [#2013](https://github.com/binwiederhier/ntfy/pull/2013))
**Bug fixes + maintenance:**
* Fix `Dockerfile-build` failing to build: use Go 1.26 and add the `cluster` package ([#1962](https://github.com/binwiederhier/ntfy/issues/1962), thanks to [@ElectricTea](https://github.com/ElectricTea) for reporting)
* Set the `org.opencontainers.image.version` label on release Docker images ([#1921](https://github.com/binwiederhier/ntfy/issues/1921)/[#1936](https://github.com/binwiederhier/ntfy/pull/1936), thanks to [@Plazzmex](https://github.com/Plazzmex) for reporting and [@w1977-0](https://github.com/w1977-0) for the fix)
* Warn at startup if `attachment-expiry-duration` is longer than `cache-duration`, since attachments never outlive their message ([#1053](https://github.com/binwiederhier/ntfy/issues/1053), thanks to [@ctschach](https://github.com/ctschach) for reporting)
* Fix PostgreSQL memory growth after v2.29.0: write message batches with one fixed `INSERT` for any batch size ([#2011](https://github.com/binwiederhier/ntfy/pull/2011))
* CLI: fix `ntfy publish` failing with "invalid header field value" when sending a multi-line message or title along with a file ([#1808](https://github.com/binwiederhier/ntfy/issues/1808), thanks to [@quantenzitrone](https://github.com/quantenzitrone) for reporting)
* Web app: fix an updated message showing next to its old version when the update arrived via the background poll instead of live ([#2010](https://github.com/binwiederhier/ntfy/pull/2010))
**Documentation:**
* Document recommended PostgreSQL pool settings for busy servers ([#2011](https://github.com/binwiederhier/ntfy/pull/2011))
## ntfy server v2.29.0
Released October 7, 2026
This release fixes how ntfy writes messages to the cache under load. On ntfy.sh, every top of the hour brought a
burst of cron-driven publishes that the single batch writer could not keep up with: messages took up to 40 seconds
to reach the database (so `poll=1` and `since=` requests missed them), and publish requests stalled behind the full
queue. The writer now stores a whole batch in one statement, publishes no longer wait on the writer, and PostgreSQL
connections are kept idle instead of re-opened on every burst. The rest is smaller fixes that accumulated since
v2.28.0, plus timezone and RFC 3339 support for delayed notifications.
**Features:**
* Support a timezone for delayed notifications via the `X-Timezone` header, `timezone` query parameter or JSON field, e.g. `At: tomorrow, 10am` with `Timezone: Asia/Tokyo`; the web app sends the browser's timezone automatically ([#1967](https://github.com/binwiederhier/ntfy/pull/1967)/[#1924](https://github.com/binwiederhier/ntfy/issues/1924), thanks to [@beemines](https://github.com/beemines) for the contribution)
* Support RFC 3339 timestamps for delayed notifications, e.g. `Delay: 2026-12-02T10:00:00Z` or `At: 2026-12-10T11:00:00-05:00`, to schedule a message at a precise point in time ([#1931](https://github.com/binwiederhier/ntfy/pull/1931), thanks to [@ALPHACOM-Brehmer](https://github.com/ALPHACOM-Brehmer) for the contribution)
**Bug fixes + maintenance:**
* Fix messages becoming visible to pollers only seconds later (and publishes stalling) under publish bursts: the message cache writer now stores each batch with a single multi-row `INSERT` instead of one statement per message (on PostgreSQL and SQLite), publishes no longer wait behind each batch write (up to 100 batches are buffered before the writer applies backpressure), and the per-publish lookup for a scheduled message with the same sequence ID is skipped when the client did not set one ([#2006](https://github.com/binwiederhier/ntfy/pull/2006))
* Fix a recurring panic in the account token-update endpoint when a concurrent anonymous request from the same IP raced the authenticated request on the shared visitor (nil-user dereference); the request now fails with HTTP 401
* Fix accepted messages being dropped from the cache on shutdown: the batched-write queue is now flushed, and `ntfy serve` shuts down gracefully on `SIGTERM`/`SIGINT` instead of dying mid-batch
* Keep PostgreSQL pool connections idle instead of re-opening them: `pool_max_idle_conns` now defaults to `pool_max_conns`, so bursts no longer pay a DNS lookup, TCP/TLS handshake and authentication per query
* Fix `since=<message-id>` replays flooding a client with a topic's entire retained history when the ID was not on the read replica yet (a client reconnecting right after receiving a message); the ID is now resolved on the primary in that case
* Use the PostgreSQL row estimate for the `messages_cached` metric instead of a full-table `COUNT(*)` every minute
* Detect Markdown when the `Content-Type` header has parameters or is uppercase, e.g. `text/markdown; charset=utf-8` ([#1995](https://github.com/binwiederhier/ntfy/pull/1995), thanks to [@cipherprofessor](https://github.com/cipherprofessor) for the contribution)
* Fix the built-in Alertmanager template showing a bogus `0001-01-01` end time for firing alerts and joining a resolved alert's start and end times onto one line ([#1946](https://github.com/binwiederhier/ntfy/pull/1946)/[#1940](https://github.com/binwiederhier/ntfy/issues/1940), thanks to [@justadityaraj](https://github.com/justadityaraj) for the contribution and [@deferred](https://github.com/deferred) for reporting)
* Ship the time zone database with the server (`time/tzdata`), so named time zones for delayed notifications work on the ARM Docker images and other minimal runtimes without system zone files
* Build with Go 1.27; the minimum Go version to build from source is now 1.26
* Groundwork for horizontal scaling: merged the first building blocks for running multiple ntfy servers as a cluster (node registry, util primitives, reserved experimental config options); clustering is not usable yet ([#1991](https://github.com/binwiederhier/ntfy/pull/1991), [#1997](https://github.com/binwiederhier/ntfy/pull/1997), [#2001](https://github.com/binwiederhier/ntfy/pull/2001), [#2003](https://github.com/binwiederhier/ntfy/pull/2003))
## ntfy server v2.28.0
Released August 27, 2026
This is a hardening release. A single topic on ntfy.sh was polled continuously with `poll=1` and no
@@ -27,7 +222,7 @@ digging into it.
* Cap a single cache replay at 10 MB of messages per topic. A poll without a `since` cursor returns a topic's entire cache, which was previously unbounded and could reach tens of megabytes on a busy topic, so one request could allocate that much on the server. The newest messages that fit are kept and a truncated response carries an `X-Messages-Truncated: 1` header
* `visitor-attachment-daily-bandwidth-limit` now also covers messages replayed from the message cache by poll requests, not just attachment traffic. A poll without a `since` cursor returns a topic's entire cache, so a topic that is cheap to fill can be re-read for many times its own size; polls beyond the budget are rejected with HTTP 429 (error code 42905) before anything is written. **Note that heavy pollers now consume the same budget as attachment downloads**, so operators serving both may want to raise the limit
### ntfy server v2.27.0
## ntfy server v2.27.0
Released August 4, 2026
This release lets you sign in with your verified email address instead of your username, which should help if you ever
@@ -86,7 +281,7 @@ especially when paired with increaseing the server-side `keepalive-interval` in
* Fix subscription icons disappearing after a few days due to Android clearing cache ([#1322](https://github.com/binwiederhier/ntfy/issues/1322), thanks to [@mcanning](https://github.com/mcanning) for reporting)
* Fix UnifiedPush `failed_reason` being sent as an enum instead of a string, which caused an exception in receiving apps that read it as a string extra ([ntfy-android#182](https://github.com/binwiederhier/ntfy-android/pull/182), thanks to [@p1gp1g](https://github.com/p1gp1g) for the contribution)
### ntfy server v2.26.3
## ntfy server v2.26.3
Released July 20, 2026
This is a hotfix release, useful pretty much only for ntfy.sh. It was adds the ability to track abusive IPs more
@@ -98,7 +293,7 @@ offenders instantly. See [ban-feed](config.md#ban-feed) for details.
* Add an abuse ban-feed: when enabled via `ban-file`, ntfy tracks a weighted strike budget per visitor and appends abusive IPs to a file that fail2ban can tail and ban on sight (`ban-file`, `ban-window`, `ban-threshold`, `ban-weights`; see [ban-feed docs](config.md#ban-feed))
### ntfy server v2.26.0
## ntfy server v2.26.0
Released July 9, 2026
This release hardens **message templates**, which are now executed with a hard-capped execution timeout. This closes
@@ -2093,6 +2288,16 @@ and the [ntfy Android app](https://github.com/binwiederhier/ntfy-android/release
## Not released yet
### ntfy server v2.36.0 (UNRELEASED)
**Features:**
* Admin API (internal, undocumented): manage tokens of other users, tiers and password reset links
**Bug fixes + maintenance:**
* Admin API: return 4xx errors instead of a 500 for invalid input, and remove a deleted user's web push subscriptions
### ntfy iOS app v1.8.0 (UNRELEASED)
**Features:**
@@ -2105,3 +2310,9 @@ and the [ntfy Android app](https://github.com/binwiederhier/ntfy-android/release
* Restore the native swipe-to-go-back gesture in the topic detail view ([ntfy-ios#45](https://github.com/binwiederhier/ntfy-ios/pull/45), thanks to [@am7590](https://github.com/am7590) for the contribution)
* Fix saved attachments being left "in use" so they couldn't be deleted in the Files app ([ntfy-ios#43](https://github.com/binwiederhier/ntfy-ios/pull/43), thanks to [@am7590](https://github.com/am7590) for the contribution)
* Improve poll request subscription matching for protected topics so notifications resolve to the real message content, with better logging ([ntfy-ios#43](https://github.com/binwiederhier/ntfy-ios/pull/43), thanks to [@am7590](https://github.com/am7590) for the contribution)
### ntfy Android v1.26.0 (UNRELEASED)
**Bug fixes + maintenance:**
* Fix copying multiple selected messages including their timestamps ([#1926](https://github.com/binwiederhier/ntfy/issues/1926), thanks to [@podtor](https://github.com/podtor) for reporting)
docs/subscribe/api.md
+2 -1
View File
@@ -324,7 +324,8 @@ Available filters (all case-insensitive):
### Subscribe to multiple topics
It's possible to subscribe to multiple topics in one HTTP call by providing a comma-separated list of topics
in the URL. This allows you to reduce the number of connections you have to maintain:
in the URL. This allows you to reduce the number of connections you have to maintain. A single subscription can
include up to 150 distinct topics; duplicate topics are ignored:
```
$ curl -s ntfy.sh/mytopic1,mytopic2/json
go.mod
+48 -47
View File
@@ -1,25 +1,25 @@
module heckel.io/ntfy/v2
go 1.25.8
go 1.26.0
require (
cloud.google.com/go/firestore v1.25.0 // indirect
cloud.google.com/go/storage v1.65.1 // indirect
cloud.google.com/go/firestore v1.26.0 // indirect
cloud.google.com/go/storage v1.69.0 // indirect
github.com/BurntSushi/toml v1.6.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
github.com/emersion/go-smtp v0.25.0
github.com/gabriel-vasile/mimetype v1.4.15
github.com/gabriel-vasile/mimetype v1.4.16
github.com/gorilla/websocket v1.5.3
github.com/mattn/go-sqlite3 v1.14.50
github.com/mattn/go-sqlite3 v1.14.52
github.com/olebedev/when v1.1.0
github.com/stretchr/testify v1.12.1
github.com/urfave/cli/v2 v2.27.7
golang.org/x/crypto v0.55.0
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sync v0.22.0
golang.org/x/term v0.45.0
golang.org/x/time v0.15.0
google.golang.org/api v0.294.0
golang.org/x/crypto v0.58.0
golang.org/x/oauth2 v0.37.0 // indirect
golang.org/x/sync v0.24.0
golang.org/x/term v0.47.0
golang.org/x/time v0.16.0
google.golang.org/api v0.306.0
gopkg.in/yaml.v2 v2.4.0
)
@@ -28,29 +28,28 @@ replace github.com/emersion/go-smtp => github.com/emersion/go-smtp v0.17.0 // Pi
require github.com/pkg/errors v0.9.1 // indirect
require (
firebase.google.com/go/v4 v4.21.0
firebase.google.com/go/v4 v4.22.0
github.com/SherClockHolmes/webpush-go v1.4.0
github.com/jackc/pgx/v5 v5.10.0
github.com/jackc/pgx/v5 v5.11.0
github.com/microcosm-cc/bluemonday v1.0.27
github.com/prometheus/client_golang v1.24.1
github.com/prometheus/client_golang v1.25.0
github.com/stripe/stripe-go/v74 v74.30.0
golang.org/x/sys v0.47.0
golang.org/x/text v0.41.0
golang.org/x/sys v0.49.0
golang.org/x/text v0.43.0
)
require (
cel.dev/expr v0.25.3 // indirect
cloud.google.com/go v0.123.0 // indirect
cloud.google.com/go/auth v0.23.2 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.13.0 // indirect
cloud.google.com/go/longrunning v1.2.0 // indirect
cloud.google.com/go/monitoring v1.30.0 // indirect
cloud.google.com/go/auth v0.24.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.3.0 // indirect
cloud.google.com/go/compute/metadata v0.10.0 // indirect
cloud.google.com/go/iam v1.14.0 // indirect
cloud.google.com/go/longrunning v1.3.0 // indirect
cloud.google.com/go/monitoring v1.31.0 // indirect
github.com/AlekSi/pointer v1.2.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.36.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.60.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.60.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 // indirect
github.com/MicahParks/keyfunc v1.9.0 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
@@ -60,46 +59,48 @@ require (
github.com/envoyproxy/go-control-plane/envoy v1.39.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
github.com/felixge/httpsnoop v1.1.0 // indirect
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
github.com/go-jose/go-jose/v4 v4.1.5 // indirect
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/s2a-go v0.1.11 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect
github.com/googleapis/gax-go/v2 v2.24.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.23 // indirect
github.com/googleapis/gax-go/v2 v2.26.2 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jackc/puddle/v2 v2.2.3 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/prometheus/client_model v0.6.3 // indirect
github.com/prometheus/common v0.72.1 // indirect
github.com/prometheus/procfs v0.22.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect
github.com/spiffe/go-spiffe/v2 v2.9.0 // indirect
github.com/stretchr/objx v0.5.3 // indirect
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.46.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect
go.opentelemetry.io/otel v1.46.0 // indirect
go.opentelemetry.io/otel/metric v1.46.0 // indirect
go.opentelemetry.io/otel/sdk v1.46.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect
go.opentelemetry.io/otel/trace v1.46.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.47.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 // indirect
go.opentelemetry.io/otel v1.47.0 // indirect
go.opentelemetry.io/otel/log v1.47.0 // indirect
go.opentelemetry.io/otel/metric v1.47.0 // indirect
go.opentelemetry.io/otel/sdk v1.47.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.47.0 // indirect
go.opentelemetry.io/otel/trace v1.47.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/net v0.61.0 // indirect
google.golang.org/appengine/v2 v2.0.6 // indirect
google.golang.org/genproto v0.0.0-20260825221802-da73d73af1c5 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 // indirect
google.golang.org/grpc v1.83.2 // indirect
google.golang.org/genproto v0.0.0-20261005182115-fad411399dd8 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20261005182115-fad411399dd8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b // indirect
google.golang.org/grpc v1.84.0 // indirect
google.golang.org/protobuf v1.36.12 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
go.sum
+102 -102
View File
@@ -2,40 +2,38 @@ cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0=
cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
cloud.google.com/go/auth v0.23.2 h1:pxSCpfiji41hpzpPdMCftEUCezpgpqmmDdYiAjCKXxo=
cloud.google.com/go/auth v0.23.2/go.mod h1:4DhBRcqvtljQN3dJ57qtqbib5ZGCYE5f2crfiiC2EM0=
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
cloud.google.com/go/firestore v1.25.0 h1:yY3rQKyQXNhnhETdseNayF6W1p4x0bdg9ZYS4hKJfOw=
cloud.google.com/go/firestore v1.25.0/go.mod h1:0PU6hj+r/QlhB6BLsRX+Kt/SYefTXrpYrBeHbYaSis8=
cloud.google.com/go/iam v1.13.0 h1:ufT3FPT5rFFXu6UtLkNoxaOaV5EuA1dsSkmemCSTo6U=
cloud.google.com/go/iam v1.13.0/go.mod h1:gHXdDEiPDvqd1q1KwBDGQlgZY/BwY760zU2LhOZS5w0=
cloud.google.com/go/logging v1.19.1 h1:7SsLhyTDBDrJw+Ll6Ns3I2mByqHXvJUc3rGjSlwiWgU=
cloud.google.com/go/logging v1.19.1/go.mod h1:2IkQ/d8jVJqV2qW8ZUGUiMjdZG1gkLD2JReGbZ8isqg=
cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM=
cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0=
cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY=
cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM=
cloud.google.com/go/storage v1.65.1 h1:LRRpBJUTf+OXDPX9jZUKZ3mSLIsz3htG+qUpeNZovyA=
cloud.google.com/go/storage v1.65.1/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM=
cloud.google.com/go/auth v0.24.1 h1:NTmd2iJb4DsGIav0KiqFMYI88nMshA2wnFMBatRmoCs=
cloud.google.com/go/auth v0.24.1/go.mod h1:1dmBXa9xfBvBr39ZnDUDkaaA91ccq9CcqENDgJA4d7c=
cloud.google.com/go/auth/oauth2adapt v0.3.0 h1:FY8oSZpCYoUNv6QxVODuMjQz4IlSOVeiQtZ08vLPz88=
cloud.google.com/go/auth/oauth2adapt v0.3.0/go.mod h1:7+2uCm7++XFO+/lN06c2HXpDXb/NMNn2/UwyBPbTnkk=
cloud.google.com/go/compute/metadata v0.10.0 h1:pyKMUQSwchgkIBBJGdILqQbs/BNJXqwSA7Ej6LAvvtY=
cloud.google.com/go/compute/metadata v0.10.0/go.mod h1:rGFHRrIif570kSibjFTMbt6/4/tzgJWFGI/HVol4GIk=
cloud.google.com/go/firestore v1.26.0 h1:7Y6wn4aj5JXl2DAsKSTpLzYKPrfrIbhgQnHDjNOJ3sQ=
cloud.google.com/go/firestore v1.26.0/go.mod h1:X7hAjktdf9wIYJEHJ/dRFpYJmpcZanf1WnWxBAq8vJE=
cloud.google.com/go/iam v1.14.0 h1:YRIB1/4xxwJ+NfI6RQr1WaOoHHeV0MizJEj940nBXJA=
cloud.google.com/go/iam v1.14.0/go.mod h1:JTPZfbHdGXJrOYREtkRoFzCDGaBdbA7ILLKO/DDuav0=
cloud.google.com/go/logging v1.20.0 h1:l7yzCB8/JSY5PHT9RsaqdT1jaBGbM6KlSJDX210HlXM=
cloud.google.com/go/logging v1.20.0/go.mod h1:ALUaJQZM691Eeujy9kFx0xREq3AMPuGd+pTRZuTYv+w=
cloud.google.com/go/longrunning v1.3.0 h1:lZ6c8SJJBmILp/Wnr3vGnBLa0q+cYfSHRCcUfVI2hpI=
cloud.google.com/go/longrunning v1.3.0/go.mod h1:wpgLxCn+UA3/TZ1bj/fW39mfDpwVlMPu1xKJyoMt7V0=
cloud.google.com/go/monitoring v1.31.0 h1:LCuAoYiq8snyL/eDAG/VC73uWFXeax8+YEfmoNGJUrY=
cloud.google.com/go/monitoring v1.31.0/go.mod h1:7uk2dIJIf7ra64dI1VuQ0ttoFqvtKjdzq46u6zNP1hU=
cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/XBM=
cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g=
cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E=
cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0=
firebase.google.com/go/v4 v4.21.0 h1:HBZV4jrLtFYj8EwWyqEZOuRLfkfkV2bpnfyyXHOhPxY=
firebase.google.com/go/v4 v4.21.0/go.mod h1:CDumIdA5oTiyDpLNVcQoW8ZrB5CTgyE2D45DuENIABg=
firebase.google.com/go/v4 v4.22.0 h1:+7ClMN69jOITseDXC7q0OKjMkRl3ZEJwZaLB900mvYs=
firebase.google.com/go/v4 v4.22.0/go.mod h1:tXyH/bqgOst15PmgHUSFaLX9woyDwclTtz/cRX+NocQ=
github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w=
github.com/AlekSi/pointer v1.2.0/go.mod h1:gZGfd3dpW4vEc/UlyfKKi1roIqcCgwOIvb0tSNSBle0=
github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.36.0 h1:3SdxXLkgAfiHRWcGTq6fneq9jgoJzneiY0yPQnjoT2E=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.36.0/go.mod h1:1iIdl0k+ppn9wT0wzR9H7HkSvIui/4qgtnKW10cQtds=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.60.0 h1:HldzheTs05E3ybqSitI/wHaof6+XERRudgZLjYbs3eE=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.60.0/go.mod h1:evkqaSczW9g2BQm1veCtgNhJ4wCCsRrOsSgNIn9LHQk=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.60.0 h1:Fx8NtDCmKH4ML2hUkPz4Dq250903vRDojMjVCDKwQuc=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.60.0/go.mod h1:V9g30lTKzfUsEW+gpWssck6u9IhARajmipodImLLcwI=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.60.0 h1:Oblia1QXBJlM/wOY9ARRUtsXdDYiMCzk3eCMikqoLbI=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.60.0/go.mod h1:SRAbhyZ4R4FagHMM9VtRgSY/lheRoht2fKelZXQUenk=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 h1:5y2LR2pripbccOte6BpZ+guv5O5W0OQIfKvUYZrGaAQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0/go.mod h1:b3f7EifFRqkVt142USoGIP62uUeZm7ALCPrs1ZjWXYo=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0 h1:tRsXhJnn0w0TNwJpVrB94wsrimzjWKFzdQlUhRdNUzc=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0/go.mod h1:ifhe5teRNcbxg9p353DhOkOgczzL/IBLYFqT1cm1zpM=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 h1:aPMa5qZU9dp4fU34o+jQgVIPFIOi20oxYg8TLigmZLg=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y=
github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o=
github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw=
github.com/SherClockHolmes/webpush-go v1.4.0 h1:ocnzNKWN23T9nvHi6IfyrQjkIc0oJWv1B1pULsf9i3s=
@@ -67,10 +65,10 @@ github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMD
github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0=
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/gabriel-vasile/mimetype v1.4.16 h1:5izNFuvA37st1dDRAcA7TtmngBmRgiIPDmTtiEvjx1E=
github.com/gabriel-vasile/mimetype v1.4.16/go.mod h1:3oFFwxX8kTklZwzIl2aQEDZT0lKY8xH92ST88bAwba4=
github.com/go-jose/go-jose/v4 v4.1.5 h1:RjgjO2LOtWOJKUC5wpwY9LR3B3vwVAz6JS2YHfYU6eA=
github.com/go-jose/go-jose/v4 v4.1.5/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -91,14 +89,14 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
github.com/google/s2a-go v0.1.11 h1:7v1V1TraNGP63WptsAkNqr2w8jgm4vM6A6qx6Lsw9iM=
github.com/google/s2a-go v0.1.11/go.mod h1:pz4tyvwXvJLLbyrkh6FW1eS2zPUXMaTmyNhYtyP2tNw=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.21 h1:OFdQ3tnCX/zaQ0Cedur3D3z7kI6HiLX9g3TiAN4/DFU=
github.com/googleapis/enterprise-certificate-proxy v0.3.21/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w=
github.com/googleapis/gax-go/v2 v2.24.0 h1:myMaPYyF9MecEmvQqMqomIwn9t/4KCZN9qnwsS76wlg=
github.com/googleapis/gax-go/v2 v2.24.0/go.mod h1:IaTHBDd7NHxSCiu0vEs8pQZu4dGZrWwuSoxCnk16OFM=
github.com/googleapis/enterprise-certificate-proxy v0.3.23 h1:J+VGF1wjAM2dU2kKUXZxJdDppRLReWP7KWcclfngQhc=
github.com/googleapis/enterprise-certificate-proxy v0.3.23/go.mod h1:Y8NO/15GNLkrXI3OR5xOFmeU3XLX1sLSdtlXgJ0zmIw=
github.com/googleapis/gax-go/v2 v2.26.2 h1:ydkmNXxj7bEmmeK5AihkKnWxyOyBR9TDebvp5L5izk8=
github.com/googleapis/gax-go/v2 v2.26.2/go.mod h1:sMKqnMesnKH+3wiRJROcttA+cJoZoGbZl1vDQ8XYtGk=
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
@@ -107,20 +105,20 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao=
github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mattn/go-sqlite3 v1.14.50 h1:dmdFvo1XG4MPzA4IkAmE9upVz/Nj31uRoM5+jC8hYbY=
github.com/mattn/go-sqlite3 v1.14.50/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U=
github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -132,20 +130,20 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0=
github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk=
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
github.com/prometheus/common v0.72.1 h1:ZAiQed/nk8IIcsdzb2TlJu23NEUDyDLQ9TXTN2WRK6Y=
github.com/prometheus/common v0.72.1/go.mod h1:1IoBY4IRaJnhs2ESjNzjilP1cJ/P8pimGoomq7zRXrA=
github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E=
github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U=
github.com/spiffe/go-spiffe/v2 v2.9.0 h1:g5qTg3qIP1HXV+GzuyEfSJFVxUCpEP5Ui1OINVJHPB4=
github.com/spiffe/go-spiffe/v2 v2.9.0/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
@@ -162,26 +160,28 @@ github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBi
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/detectors/gcp v1.46.0 h1:PI8dGkqDaQkwJ8kOopqMhDTbrnK3UIeG/RCHH4HErbo=
go.opentelemetry.io/contrib/detectors/gcp v1.46.0/go.mod h1:nsrN5c/sOLoY2vsPxN/rQ0V0nvGrWJCqcW4UXLtqNG8=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0 h1:hqxVTu/GtBF+vJ8d1fzW7fRxZFvgoDjWcxwwCaFDYpU=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0/go.mod h1:z5fVEF4X5v0ESvlJqBrrFlBVoj5EQuefZpzsu7R+x5Q=
go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=
go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU=
go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4=
go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=
go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=
go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=
go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=
go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
go.opentelemetry.io/contrib/detectors/gcp v1.47.0 h1:Ct6+aaiSO6Nk5SOb6PYcR7iWp+83YF/pL1OJcGqOmbg=
go.opentelemetry.io/contrib/detectors/gcp v1.47.0/go.mod h1:3N259ss0b1AVOs+fYw7lOO0mcUwNqSIpQ+Zitpuqx00=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0/go.mod h1:A+EVhDakAj1waJBPhmTszZaSr08ww8MRRKy1nB1rqck=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0 h1:dm9iyzn6tioYZtwqaiBSU0TSI8Yu/8dTIbfG0+B49DY=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0/go.mod h1:xAvxYjYK28qvt+yu4BYZ/zMmAjwMXINXD6JiMyeB8iI=
go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI=
go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw=
go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY=
go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU=
go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM=
go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k=
go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I=
go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0=
go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw=
go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw=
go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
@@ -194,8 +194,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -210,10 +210,10 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw=
golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -221,8 +221,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -235,8 +235,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -246,8 +246,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -259,10 +259,10 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
@@ -273,18 +273,18 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/api v0.294.0 h1:8gASjJxdtcIieB3OqbkLcF0FfbXVNqKtU5iozD1ssvA=
google.golang.org/api v0.294.0/go.mod h1:02qB8+Ox1ZFzcaKFMguy1nQLJmSIyvV6Ff4txJEXtl4=
google.golang.org/api v0.306.0 h1:jbakJtAJAO1d8PUFandqfYHuf/ecaeTgoas7+hpXLHc=
google.golang.org/api v0.306.0/go.mod h1:4XMH+b/ZEfuTbK7FnrFRY9c4xfFHVflGTKuy/i5wUmE=
google.golang.org/appengine/v2 v2.0.6 h1:LvPZLGuchSBslPBp+LAhihBeGSiRh1myRoYK4NtuBIw=
google.golang.org/appengine/v2 v2.0.6/go.mod h1:WoEXGoXNfa0mLvaH5sV3ZSGXwVmy8yf7Z1JKf3J3wLI=
google.golang.org/genproto v0.0.0-20260825221802-da73d73af1c5 h1:jPP56YzdY899KJ5W7efXHt/CkjlVfAaoFOwdi/IEAFA=
google.golang.org/genproto v0.0.0-20260825221802-da73d73af1c5/go.mod h1:gutZdP0DwAHp4vu5WaXgEK7tjsJ77ZEqzlOFWGZGziE=
google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5 h1:izFU9hz7aeLI/Mi1J0991ae+xcwRLr7hTqWnB/9aIIU=
google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5/go.mod h1:3LhxRw4YYkf+ylAfgaY9JlVLFKhokkCV8duhLLe7+t0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 h1:1VUiZAXyC+zmiFYi+WLtBzr68Cj8wOofHjjrA/kkizc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA=
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
google.golang.org/genproto v0.0.0-20261005182115-fad411399dd8 h1:19ViM1+Odqvg4xpcB+8U1fTpy23WB3sMMyR3LSsJUpI=
google.golang.org/genproto v0.0.0-20261005182115-fad411399dd8/go.mod h1:+ilrBox3TDCuZtI7U55k3p0uUJ/yNkMUkS1rVg+yxkg=
google.golang.org/genproto/googleapis/api v0.0.0-20261005182115-fad411399dd8 h1:cFbkNigDInw+wVTXc+CyhUsoHGJCw0BR6mlNoRVfre0=
google.golang.org/genproto/googleapis/api v0.0.0-20261005182115-fad411399dd8/go.mod h1:QK6p7IgD7IZEZJBhhSUCib5X2vMOzlrGCGUw2XyVDh0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b h1:6WNWHZk91dp4gZ25642OALkuG1UVNZbwENJD19DnB8k=
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc=
google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0=
google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
limit/service.go
+285
View File
@@ -0,0 +1,285 @@
// Package limit answers one question: may key K use N units of kind X? A key is a visitor
// identity (see package visitor), a kind is what is being counted (requests, messages, bytes, ...),
// and a Rule, passed in on every call, says how much is allowed: a token bucket, a daily cap, or
// both. Rules are not baked into the state, so a changed rule (a tier change) applies on the next
// call without rebuilding anything and without handing back a full bucket.
//
// The service knows nothing about users, tiers or HTTP. All state is in memory, sharded by key.
package limit
import (
"math"
"sync"
"time"
"heckel.io/ntfy/v2/util"
)
const (
shardCount = 64 // Keys are spread over this many independently locked maps
)
// Service keeps the per-key state and decides on usage. See the package comment.
type Service struct {
conf *Config
now func() time.Time // Injectable clock, for tests
shards [shardCount]*shard
}
// shard is one independently locked part of the key space
type shard struct {
entries map[string]*entry
mu sync.Mutex // Protects entries and everything inside them
}
// entry is one key's state: a bucket and a daily count per kind, and when it was last used
type entry struct {
buckets [numKinds]bucket
counts [numKinds]int64 // Units used today, per kind
seen int64 // Unix nanoseconds of the last use, for pruning
}
// bucket is a token bucket. It is created full: last is zero until the first use, and tokens
// then start at the rule's burst. Tokens may go negative (debt) when peers' usage is charged in.
type bucket struct {
tokens float64
last int64 // Unix nanoseconds of the last refill, 0 if never used
burst int64 // Burst of the rule the bucket was last used with; caps refunds and debt
}
// New creates the service
func New(conf *Config) *Service {
return newService(conf, time.Now)
}
func newService(conf *Config, now func() time.Time) *Service {
s := &Service{
conf: conf,
now: now,
}
for i := range s.shards {
s.shards[i] = &shard{
entries: make(map[string]*entry),
}
}
return s
}
// Allow decides whether the key may use n units of the rule's kind now, and if so, uses them.
// Both the bucket (if the rule has one) and the daily cap (if it has one) must have room.
func (s *Service) Allow(key string, rule Rule, n int64) Decision {
now := s.now()
sh := s.shards[shardIndex(key)]
sh.mu.Lock()
defer sh.mu.Unlock()
e := sh.entryNoLock(key, now)
e.seen = now.UnixNano()
b := &e.buckets[rule.Kind]
if rule.Burst > 0 {
b.refill(now, &rule)
if n > rule.Burst {
return Decision{} // Can never fit, so there is no time to wait for
} else if b.tokens < float64(n) {
return Decision{RetryAfter: b.wait(n, &rule)}
}
}
if rule.Daily > 0 && e.counts[rule.Kind]+n > rule.Daily {
return Decision{RetryAfter: untilReset(now, s.conf.ResetTime)}
}
if rule.Burst > 0 {
b.tokens -= float64(n)
}
e.counts[rule.Kind] += n
return Decision{OK: true}
}
// Tokens returns the whole tokens the key's bucket for the rule's kind has right now, without
// using any. It is what a log line or a "may I try" check wants; Allow is what consumes.
func (s *Service) Tokens(key string, rule Rule) int64 {
if rule.Burst <= 0 {
return 0
}
now := s.now()
sh := s.shards[shardIndex(key)]
sh.mu.Lock()
defer sh.mu.Unlock()
e, ok := sh.entries[key]
if !ok {
return rule.Burst // A fresh bucket is full; no need to create it to say so
}
b := &e.buckets[rule.Kind]
b.refill(now, &rule)
return int64(math.Max(0, math.Floor(b.tokens)))
}
// Refund hands n units of the kind back to the key, e.g. for bytes that were reserved but not
// sent. The bucket cannot exceed its burst, and the daily count cannot go below zero.
func (s *Service) Refund(key string, kind Kind, n int64) {
sh := s.shards[shardIndex(key)]
sh.mu.Lock()
defer sh.mu.Unlock()
e, ok := sh.entries[key]
if !ok {
return
}
b := &e.buckets[kind]
if b.last != 0 {
b.tokens = math.Min(b.tokens+float64(n), float64(b.burst))
}
e.counts[kind] = max(0, e.counts[kind]-n)
}
// Usage returns the key's usage today, of every kind
func (s *Service) Usage(key string) Usage {
sh := s.shards[shardIndex(key)]
sh.mu.Lock()
defer sh.mu.Unlock()
e, ok := sh.entries[key]
if !ok {
return Usage{}
}
return Usage{counts: e.counts}
}
// Seed sets the key's daily counts, but only if the key is unknown to the service. It carries
// counts persisted elsewhere (the user's stats) into a fresh key.
func (s *Service) Seed(key string, usage Usage) {
now := s.now()
sh := s.shards[shardIndex(key)]
sh.mu.Lock()
defer sh.mu.Unlock()
if _, ok := sh.entries[key]; ok {
return
}
e := sh.entryNoLock(key, now)
e.counts = usage.counts
}
// ResetUsage zeroes the daily counts of every key (the daily reset)
func (s *Service) ResetUsage() {
for _, sh := range s.shards {
sh.mu.Lock()
for _, e := range sh.entries {
e.counts = [numKinds]int64{}
}
sh.mu.Unlock()
}
}
// Prune forgets keys that have not been used for longer than idle, and returns how many. A key
// that comes back starts with full buckets and zero counts.
func (s *Service) Prune(idle time.Duration) int {
cutoff := s.now().Add(-idle).UnixNano()
pruned := 0
for _, sh := range s.shards {
sh.mu.Lock()
for key, e := range sh.entries {
if e.seen < cutoff {
delete(sh.entries, key)
pruned++
}
}
sh.mu.Unlock()
}
return pruned
}
// Touch marks the key as used now, so Prune keeps it, e.g. for a long-lived subscription
func (s *Service) Touch(key string) {
now := s.now()
sh := s.shards[shardIndex(key)]
sh.mu.Lock()
defer sh.mu.Unlock()
sh.entryNoLock(key, now).seen = now.UnixNano()
}
// Seen returns when the key was last used, or the zero time if the service does not know it
func (s *Service) Seen(key string) time.Time {
sh := s.shards[shardIndex(key)]
sh.mu.Lock()
defer sh.mu.Unlock()
e, ok := sh.entries[key]
if !ok {
return time.Time{}
}
return time.Unix(0, e.seen).UTC()
}
// Limiter returns a util.Limiter view of the key's bucket for the rule, for the readers and
// writers in util that meter bytes. A negative AllowN refunds.
func (s *Service) Limiter(key string, rule Rule) util.Limiter {
return &limiter{
s: s,
key: key,
rule: rule,
}
}
// Size returns the number of keys the service holds
func (s *Service) Size() int {
size := 0
for _, sh := range s.shards {
sh.mu.Lock()
size += len(sh.entries)
sh.mu.Unlock()
}
return size
}
// reset fills the key's bucket of the kind and zeroes its count (see limiter.Reset)
func (s *Service) reset(key string, kind Kind) {
sh := s.shards[shardIndex(key)]
sh.mu.Lock()
defer sh.mu.Unlock()
e, ok := sh.entries[key]
if !ok {
return
}
e.buckets[kind].tokens = float64(e.buckets[kind].burst)
e.counts[kind] = 0
}
// entryNoLock returns the key's entry, creating it if needed. The caller must hold sh.mu.
func (sh *shard) entryNoLock(key string, now time.Time) *entry {
e, ok := sh.entries[key]
if !ok {
e = &entry{seen: now.UnixNano()}
sh.entries[key] = e
}
return e
}
// refill adds the tokens earned since the last refill, capped at the rule's burst, and remembers
// the burst. A bucket that was never used starts full.
func (b *bucket) refill(now time.Time, rule *Rule) {
nowNanos := now.UnixNano()
if b.last == 0 {
b.tokens = float64(rule.Burst)
} else {
if rule.Burst != b.burst {
// A changed burst (tier change) keeps what was spent: shift the tokens by the
// difference, down to a debt of one burst
b.tokens = math.Max(b.tokens+float64(rule.Burst-b.burst), -float64(rule.Burst))
}
// A timestamp older than the last refill (a call that read the clock before another one
// took the lock, or a wall clock set back) neither charges nor moves the refill point back
if rule.Refill > 0 && nowNanos > b.last {
b.tokens += float64(nowNanos-b.last) / float64(rule.Refill)
}
}
if b.tokens > float64(rule.Burst) {
b.tokens = float64(rule.Burst)
}
if nowNanos > b.last {
b.last = nowNanos
}
b.burst = rule.Burst
}
// wait returns how long until the bucket has n tokens, or 0 if it never will (no refill)
func (b *bucket) wait(n int64, rule *Rule) time.Duration {
if rule.Refill <= 0 {
return 0
}
return time.Duration((float64(n) - b.tokens) * float64(rule.Refill))
}
limit/service_test.go
+610
View File
@@ -0,0 +1,610 @@
package limit
import (
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// testClock is a clock the tests move by hand
type testClock struct {
t time.Time
mu sync.Mutex
}
func (c *testClock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.t
}
func (c *testClock) advance(d time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.t = c.t.Add(d)
}
func newTestServiceWithClock(t *testing.T) (*Service, *testClock) {
t.Helper()
clock := &testClock{t: time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC)}
conf := &Config{ResetTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC)}
return newService(conf, clock.now), clock
}
func newTestService(t *testing.T) *Service {
t.Helper()
s, _ := newTestServiceWithClock(t)
return s
}
func TestService_Allow_Bucket(t *testing.T) {
s, clock := newTestServiceWithClock(t)
rule := Rule{
Kind: KindRequests,
Burst: 3,
Refill: 100 * time.Millisecond,
}
for i := 0; i < 3; i++ {
require.True(t, s.Allow("k", rule, 1).OK)
}
d := s.Allow("k", rule, 1)
require.False(t, d.OK)
require.Equal(t, 100*time.Millisecond, d.RetryAfter)
clock.advance(50 * time.Millisecond)
d = s.Allow("k", rule, 1)
require.False(t, d.OK)
require.Equal(t, 50*time.Millisecond, d.RetryAfter)
clock.advance(50 * time.Millisecond)
require.True(t, s.Allow("k", rule, 1).OK)
require.False(t, s.Allow("k", rule, 1).OK)
// A full minute refills the bucket, but never past the burst
clock.advance(time.Minute)
require.Equal(t, int64(3), s.Tokens("k", rule))
for i := 0; i < 3; i++ {
require.True(t, s.Allow("k", rule, 1).OK)
}
require.False(t, s.Allow("k", rule, 1).OK)
// Other keys are independent
require.True(t, s.Allow("other", rule, 1).OK)
}
func TestService_Allow_Daily(t *testing.T) {
s, _ := newTestServiceWithClock(t) // Noon UTC, 12 hours before the reset
rule := Rule{
Kind: KindMessages,
Daily: 2,
}
require.True(t, s.Allow("k", rule, 1).OK)
require.True(t, s.Allow("k", rule, 1).OK)
d := s.Allow("k", rule, 1)
require.False(t, d.OK)
require.Equal(t, 12*time.Hour, d.RetryAfter)
require.Equal(t, int64(2), s.Usage("k").Count(KindMessages))
// Denied usage is not counted
require.Equal(t, int64(2), s.Usage("k").Count(KindMessages))
// The reset starts the day over
s.ResetUsage()
require.Equal(t, int64(0), s.Usage("k").Count(KindMessages))
require.True(t, s.Allow("k", rule, 1).OK)
}
func TestService_Allow_BucketAndDaily(t *testing.T) {
// Emails: a burst of 2 refilling slowly, and a daily cap of 3
s, clock := newTestServiceWithClock(t)
rule := Rule{
Kind: KindEmails,
Burst: 2,
Refill: time.Minute,
Daily: 3,
}
require.True(t, s.Allow("k", rule, 1).OK)
require.True(t, s.Allow("k", rule, 1).OK)
d := s.Allow("k", rule, 1)
require.False(t, d.OK)
require.Equal(t, time.Minute, d.RetryAfter) // The bucket is what denies
clock.advance(time.Minute)
require.True(t, s.Allow("k", rule, 1).OK)
clock.advance(time.Hour)
d = s.Allow("k", rule, 1)
require.False(t, d.OK)
require.Equal(t, 11*time.Hour-time.Minute, d.RetryAfter) // The daily cap is what denies now
require.Equal(t, int64(3), s.Usage("k").Count(KindEmails))
}
func TestService_Allow_N(t *testing.T) {
s, clock := newTestServiceWithClock(t)
rule := Rule{
Kind: KindBandwidth,
Burst: 1000,
Refill: time.Millisecond,
} // 1 kB burst, 1 kB/s
require.True(t, s.Allow("k", rule, 600).OK)
d := s.Allow("k", rule, 600)
require.False(t, d.OK)
require.Equal(t, 200*time.Millisecond, d.RetryAfter)
clock.advance(200 * time.Millisecond)
require.True(t, s.Allow("k", rule, 600).OK)
require.Equal(t, int64(1200), s.Usage("k").Count(KindBandwidth))
// More than the burst can never be allowed, so there is no retry time
d = s.Allow("k", rule, 1001)
require.False(t, d.OK)
require.Equal(t, time.Duration(0), d.RetryAfter)
}
func TestService_Allow_RuleChangeKeepsSpent(t *testing.T) {
// A tier change passes a new rule on the next call: what was spent stays spent, so a bigger
// burst adds its difference and a smaller one takes it away (down to a debt of one burst)
s, _ := newTestServiceWithClock(t)
rule := Rule{
Kind: KindRequests,
Burst: 10,
Refill: time.Hour,
}
for i := 0; i < 8; i++ {
require.True(t, s.Allow("k", rule, 1).OK)
}
bigger := Rule{
Kind: KindRequests,
Burst: 100,
Refill: time.Hour,
}
require.Equal(t, int64(92), s.Tokens("k", bigger)) // 8 spent
require.True(t, s.Allow("k", bigger, 2).OK) // 10 spent
smaller := Rule{
Kind: KindRequests,
Burst: 5,
Refill: time.Hour,
}
require.Equal(t, int64(0), s.Tokens("k", smaller)) // 10 spent of 5: in debt
require.False(t, s.Allow("k", smaller, 1).OK)
require.Equal(t, int64(90), s.Tokens("k", bigger)) // Back up: still 10 spent
// A fresh key with the smaller rule starts with its burst
require.Equal(t, int64(5), s.Tokens("fresh", smaller))
for i := 0; i < 5; i++ {
require.True(t, s.Allow("fresh", smaller, 1).OK)
}
require.False(t, s.Allow("fresh", smaller, 1).OK)
require.Equal(t, int64(95), s.Tokens("fresh", bigger)) // 5 spent
}
func TestService_Allow_DailyChangeAppliesAtOnce(t *testing.T) {
s, _ := newTestServiceWithClock(t)
require.True(t, s.Allow("k", Rule{
Kind: KindMessages,
Daily: 5,
}, 5).OK)
require.False(t, s.Allow("k", Rule{
Kind: KindMessages,
Daily: 5,
}, 1).OK)
require.True(t, s.Allow("k", Rule{
Kind: KindMessages,
Daily: 6,
}, 1).OK) // Upgraded
require.False(t, s.Allow("k", Rule{
Kind: KindMessages,
Daily: 3,
}, 1).OK) // Downgraded below the count
require.Equal(t, int64(6), s.Usage("k").Count(KindMessages))
}
func TestService_Allow_Unlimited(t *testing.T) {
s, _ := newTestServiceWithClock(t)
for i := 0; i < 1000; i++ {
require.True(t, s.Allow("k", Rule{Kind: KindTopics}, 1).OK)
}
require.Equal(t, int64(1000), s.Usage("k").Count(KindTopics))
}
func TestService_Allow_NeverRefills(t *testing.T) {
s, clock := newTestServiceWithClock(t)
rule := Rule{
Kind: KindAccount,
Burst: 2,
} // No refill
require.True(t, s.Allow("k", rule, 1).OK)
require.True(t, s.Allow("k", rule, 1).OK)
clock.advance(24 * time.Hour)
d := s.Allow("k", rule, 1)
require.False(t, d.OK)
require.Equal(t, time.Duration(0), d.RetryAfter)
}
func TestService_Refund(t *testing.T) {
s, _ := newTestServiceWithClock(t)
rule := Rule{
Kind: KindBandwidth,
Burst: 3,
Refill: time.Hour,
Daily: 100,
}
require.True(t, s.Allow("k", rule, 3).OK)
require.False(t, s.Allow("k", rule, 1).OK)
s.Refund("k", KindBandwidth, 1)
require.True(t, s.Allow("k", rule, 1).OK)
require.False(t, s.Allow("k", rule, 1).OK)
require.Equal(t, int64(3), s.Usage("k").Count(KindBandwidth))
// A refund cannot exceed the burst, nor make the count negative
s.Refund("k", KindBandwidth, 100)
require.Equal(t, int64(3), s.Tokens("k", rule))
require.Equal(t, int64(0), s.Usage("k").Count(KindBandwidth))
// Refunding an unknown key or kind is harmless
s.Refund("unknown", KindBandwidth, 1)
require.Equal(t, 1, s.Size())
}
func TestService_Tokens_DoesNotConsume(t *testing.T) {
s, _ := newTestServiceWithClock(t)
rule := Rule{
Kind: KindAuth,
Burst: 5,
Refill: time.Minute,
}
require.Equal(t, int64(5), s.Tokens("k", rule))
require.Equal(t, int64(5), s.Tokens("k", rule))
require.True(t, s.Allow("k", rule, 2).OK)
require.Equal(t, int64(3), s.Tokens("k", rule))
require.Equal(t, int64(0), s.Tokens("k", Rule{Kind: KindAuth})) // No bucket, no tokens
}
func TestService_Seed(t *testing.T) {
s, _ := newTestServiceWithClock(t)
var usage Usage
usage.Set(KindMessages, 120)
usage.Set(KindEmails, 7)
s.Seed("k", usage)
require.Equal(t, int64(120), s.Usage("k").Count(KindMessages))
require.Equal(t, int64(7), s.Usage("k").Count(KindEmails)) // Every kind at once, not only the first
usage.Set(KindMessages, 999)
s.Seed("k", usage) // Known key: ignored
require.Equal(t, int64(120), s.Usage("k").Count(KindMessages))
require.False(t, s.Allow("k", Rule{
Kind: KindMessages,
Daily: 120,
}, 1).OK)
require.True(t, s.Allow("k", Rule{
Kind: KindMessages,
Daily: 121,
}, 1).OK)
}
func TestService_Prune(t *testing.T) {
s, clock := newTestServiceWithClock(t)
rule := Rule{
Kind: KindRequests,
Burst: 1,
Refill: time.Hour,
}
require.True(t, s.Allow("old", rule, 1).OK)
clock.advance(10 * time.Minute)
require.True(t, s.Allow("fresh", rule, 1).OK)
require.Equal(t, 2, s.Size())
s.Prune(5 * time.Minute)
require.Equal(t, 1, s.Size())
require.True(t, s.Allow("old", rule, 1).OK) // Forgotten: back with a full bucket
require.False(t, s.Allow("fresh", rule, 1).OK) // Kept
}
func TestService_Allow_Concurrent(t *testing.T) {
s := New(&Config{ResetTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC)})
rule := Rule{
Kind: KindRequests,
Burst: 500,
Refill: time.Hour,
Daily: 400,
}
var wg sync.WaitGroup
var admitted sync.Map
for g := 0; g < 16; g++ {
wg.Add(1)
go func(g int) {
defer wg.Done()
n := 0
for i := 0; i < 100; i++ {
if s.Allow("hot", rule, 1).OK {
n++
}
s.Allow(fmt.Sprintf("cold-%d-%d", g, i), rule, 1)
}
admitted.Store(g, n)
}(g)
}
wg.Wait()
total := 0
admitted.Range(func(_, v any) bool {
total += v.(int)
return true
})
require.Equal(t, 400, total) // The daily cap, exactly once
require.Equal(t, int64(400), s.Usage("hot").Count(KindRequests))
require.Equal(t, 1601, s.Size())
}
func BenchmarkAllow_HotKey(b *testing.B) {
s := New(&Config{ResetTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC)})
rule := Rule{
Kind: KindRequests,
Burst: 60,
Refill: 5 * time.Second,
Daily: 1 << 40,
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
s.Allow("ip:1.2.3.4", rule, 1)
}
}
func BenchmarkAllow_ManyKeys(b *testing.B) {
s := New(&Config{ResetTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC)})
rule := Rule{
Kind: KindRequests,
Burst: 60,
Refill: 5 * time.Second,
}
keys := make([]string, 100000)
for i := range keys {
keys[i] = fmt.Sprintf("ip:10.%d.%d.%d", i>>16, (i>>8)&255, i&255)
s.Allow(keys[i], rule, 1)
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
s.Allow(keys[i%len(keys)], rule, 1)
}
}
func BenchmarkAllow_Parallel(b *testing.B) {
s := New(&Config{ResetTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC)})
rule := Rule{
Kind: KindRequests,
Burst: 60,
Refill: 5 * time.Second,
}
b.ReportAllocs()
b.RunParallel(func(pb *testing.PB) {
i := 0
for pb.Next() {
s.Allow(fmt.Sprintf("ip:10.0.%d.%d", (i>>8)&255, i&255), rule, 1)
i++
}
})
}
func TestService_TouchAndSeen(t *testing.T) {
s, clock := newTestServiceWithClock(t)
require.True(t, s.Seen("k").IsZero())
s.Touch("k")
require.Equal(t, clock.now(), s.Seen("k"))
clock.advance(10 * time.Minute)
s.Touch("k")
s.Prune(5 * time.Minute)
require.Equal(t, 1, s.Size()) // Touched, so not idle
require.Equal(t, clock.now(), s.Seen("k"))
clock.advance(10 * time.Minute)
s.Prune(5 * time.Minute)
require.True(t, s.Seen("k").IsZero())
}
func TestService_FirebaseKind(t *testing.T) {
kind, ok := ParseKind("firebase")
require.True(t, ok)
require.Equal(t, KindFirebase, kind)
}
func TestService_ResetUsage_KeepsBuckets(t *testing.T) {
// The daily reset starts the counts over, but the buckets keep their tokens: an email bucket
// emptied right before the reset is not refilled by it (it used to be)
s, _ := newTestServiceWithClock(t)
rule := Rule{
Kind: KindEmails,
Burst: 2,
Refill: time.Hour,
Daily: 3,
}
require.True(t, s.Allow("k", rule, 2).OK)
s.ResetUsage()
require.Equal(t, int64(0), s.Usage("k").Count(KindEmails))
require.Equal(t, int64(0), s.Tokens("k", rule))
d := s.Allow("k", rule, 1)
require.False(t, d.OK)
require.Equal(t, time.Hour, d.RetryAfter) // The bucket is what denies, not the day
}
// Fails if Allow stops debiting bucket tokens; weighted concurrent calls must share one burst.
func TestService_Allow_ConcurrentBucketOnly(t *testing.T) {
s := newTestService(t)
rule := Rule{
Kind: KindBandwidth,
Burst: 97,
}
start := make(chan struct{})
var wg sync.WaitGroup
var admitted atomic.Int64
for g := 0; g < 32; g++ {
wg.Add(1)
go func(g int) {
defer wg.Done()
<-start
for i := 0; i < 64; i++ {
n := int64(1 + g%7)
if s.Allow("hot", rule, n).OK {
admitted.Add(n)
}
}
}(g)
}
close(start)
wg.Wait()
require.Equal(t, rule.Burst, admitted.Load())
require.Equal(t, admitted.Load(), s.Usage("hot").Count(rule.Kind))
require.Zero(t, s.Tokens("hot", rule))
}
// Fails if ResetUsage also clears buckets; a reset racing with publishes grants only one new daily allowance.
func TestService_ResetUsage_ConcurrentAllow(t *testing.T) {
for round := 0; round < 20; round++ {
s := newTestService(t)
rule := Rule{
Kind: KindMessages,
Burst: 81,
Daily: 40,
}
require.True(t, s.Allow("hot", rule, 40).OK)
start := make(chan struct{})
var wg sync.WaitGroup
var admitted atomic.Int64
for g := 0; g < 16; g++ {
wg.Add(1)
go func() {
defer wg.Done()
<-start
for i := 0; i < 40; i++ {
if s.Allow("hot", rule, 1).OK {
admitted.Add(1)
}
}
}()
}
wg.Add(1)
go func() {
defer wg.Done()
<-start
s.ResetUsage()
}()
close(start)
wg.Wait()
for i := 0; i < 40; i++ {
if s.Allow("hot", rule, 1).OK {
admitted.Add(1)
}
}
require.Equal(t, int64(40), admitted.Load())
require.Equal(t, admitted.Load(), s.Usage("hot").Count(rule.Kind))
require.Equal(t, int64(1), s.Tokens("hot", rule))
}
}
// Fails if Allow no longer refreshes seen; pruning must retain an active exhausted key while deleting idle keys.
func TestService_Prune_ConcurrentAllow(t *testing.T) {
s, clock := newTestServiceWithClock(t)
rule := Rule{
Kind: KindRequests,
Burst: 1,
}
require.True(t, s.Allow("hot", rule, 1).OK)
for i := 0; i < 128; i++ {
require.True(t, s.Allow(fmt.Sprintf("idle-%d", i), rule, 1).OK)
}
clock.advance(2 * time.Hour)
require.False(t, s.Allow("hot", rule, 1).OK)
start := make(chan struct{})
var wg sync.WaitGroup
var admitted atomic.Int64
for g := 0; g < 8; g++ {
wg.Add(1)
go func() {
defer wg.Done()
<-start
for i := 0; i < 100; i++ {
if s.Allow("hot", rule, 1).OK {
admitted.Add(1)
}
}
}()
}
wg.Add(1)
go func() {
defer wg.Done()
<-start
for i := 0; i < 100; i++ {
s.Prune(time.Hour)
}
}()
close(start)
wg.Wait()
require.Zero(t, admitted.Load())
require.Equal(t, int64(1), s.Usage("hot").Count(rule.Kind))
require.Equal(t, 1, s.Size())
}
// Fails if Seed overwrites existing entries, including a key known only through another kind or a daily reset.
func TestService_Seed_ConcurrentKnownKey(t *testing.T) {
s := newTestService(t)
require.True(t, s.Allow("hot", Rule{Kind: KindRequests}, 1).OK)
s.ResetUsage()
var usage Usage
usage.Set(KindMessages, 9999)
start := make(chan struct{})
var wg sync.WaitGroup
for g := 0; g < 16; g++ {
wg.Add(1)
go func() {
defer wg.Done()
<-start
for i := 0; i < 100; i++ {
s.Seed("hot", usage)
s.Allow("hot", Rule{Kind: KindMessages}, 1)
}
}()
}
close(start)
wg.Wait()
require.Equal(t, int64(1600), s.Usage("hot").Count(KindMessages))
require.Zero(t, s.Usage("hot").Count(KindRequests))
}
// Fails if refill is disabled; nanosecond and century intervals must earn exactly one token per interval.
func TestService_Allow_ExtremeRefill(t *testing.T) {
for _, interval := range []time.Duration{time.Nanosecond, 100 * 365 * 24 * time.Hour} {
t.Run(interval.String(), func(t *testing.T) {
s, clock := newTestServiceWithClock(t)
rule := Rule{
Kind: KindRequests,
Burst: 2,
Refill: interval,
}
require.True(t, s.Allow("hot", rule, 2).OK)
d := s.Allow("hot", rule, 1)
require.False(t, d.OK)
require.Equal(t, interval, d.RetryAfter)
clock.advance(interval)
require.True(t, s.Allow("hot", rule, 1).OK)
require.False(t, s.Allow("hot", rule, 1).OK)
})
}
}
func TestService_Allow_ClockGoingBackDoesNotCharge(t *testing.T) {
// A call that carries an older timestamp than the bucket's last refill (a goroutine paused
// between reading the clock and taking the shard lock, or a wall clock set back) must not
// take tokens away, and must not move the refill point back either (that would credit the
// same time twice)
s, clock := newTestServiceWithClock(t)
rule := Rule{
Kind: KindRequests,
Burst: 3,
Refill: time.Second,
}
require.True(t, s.Allow("k", rule, 1).OK)
require.True(t, s.Allow("k", rule, 1).OK)
clock.advance(-time.Second)
require.True(t, s.Allow("k", rule, 1).OK) // The third token is still there
require.False(t, s.Allow("k", rule, 1).OK)
clock.advance(time.Second) // Back to where the bucket was last refilled: no time has passed
require.False(t, s.Allow("k", rule, 1).OK)
clock.advance(time.Second)
require.True(t, s.Allow("k", rule, 1).OK)
}
limit/types.go
+116
View File
@@ -0,0 +1,116 @@
package limit
import (
"time"
)
// Kind is what a key is being limited on: a request, a message, an email, ... Each kind has its
// own token bucket and daily count per key.
type Kind uint8
// The kinds of usage a key is limited on
const (
KindRequests Kind = iota // Requests of (almost) any type, including publishes
KindMessages // Published messages
KindEmails // Emails sent on behalf of the key
KindCalls // Phone calls made on behalf of the key
KindBandwidth // Attachment and poll bytes, up and down
KindTopics // Topics created in the server's in-memory topic map
KindAccount // Account actions: signups and password reset requests
KindAuth // Failed authentication attempts
KindFirebase // Firebase sends, paused for a while after Firebase reports the quota exceeded
numKinds // Sentinel, keep last
)
// kindNames are the stable names of the kinds, used in the usage table and on the wire; they
// must never change once released
var kindNames = [numKinds]string{"requests", "messages", "emails", "calls", "bandwidth", "topics", "account", "auth", "firebase"}
// Rule says how much of a kind a key may use: a token bucket of Burst tokens refilling one token
// per Refill, a Daily cap, or both. A zero Rule allows everything.
type Rule struct {
Kind Kind
Burst int64 // Token bucket size; 0 for no bucket
Refill time.Duration // Time to refill one token; 0 for a bucket that never refills
Daily int64 // Daily cap; 0 for none
}
// Decision is the answer to Allow: whether the usage was admitted, and if not, how long until
// it could be (0 if unknown, e.g. when n exceeds the burst)
type Decision struct {
OK bool
RetryAfter time.Duration
}
// Usage is a key's usage of every kind today
type Usage struct {
counts [numKinds]int64
}
// Config configures the service. ResetTime is the wall-clock time of day (UTC) at which daily
// counts start over.
type Config struct {
ResetTime time.Time
}
// limiter is the util.Limiter view of one key's bucket for one rule (see Service.Limiter)
type limiter struct {
s *Service
key string
rule Rule
}
// String returns the kind's stable name
func (k Kind) String() string {
if k < numKinds {
return kindNames[k]
}
return "unknown"
}
// ParseKind returns the kind with the given name
func ParseKind(name string) (Kind, bool) {
for k, n := range kindNames {
if n == name {
return Kind(k), true
}
}
return 0, false
}
// Count returns the usage of the given kind
func (u Usage) Count(kind Kind) int64 {
return u.counts[kind]
}
// Set sets the usage of the given kind (see Service.Seed)
func (u *Usage) Set(kind Kind, count int64) {
u.counts[kind] = count
}
// Allow uses one unit, or returns false if the limit has been reached
func (l *limiter) Allow() bool {
return l.AllowN(1)
}
// AllowN uses n units, or returns false if the limit has been reached. A negative n refunds, which
// is how util.LimitReader and util.LimitWriter revert a limiter when a later one rejects.
func (l *limiter) AllowN(n int64) bool {
if n < 0 {
l.s.Refund(l.key, l.rule.Kind, -n)
return true
} else if n == 0 {
return true
}
return l.s.Allow(l.key, l.rule, n).OK
}
// Value returns the key's usage of the kind today
func (l *limiter) Value() int64 {
return l.s.Usage(l.key).Count(l.rule.Kind)
}
// Reset fills the bucket and zeroes the key's usage of the kind
func (l *limiter) Reset() {
l.s.reset(l.key, l.rule.Kind)
}
limit/types_test.go
+54
View File
@@ -0,0 +1,54 @@
package limit
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestKind_String_ParseKind(t *testing.T) {
for k := Kind(0); k < numKinds; k++ {
parsed, ok := ParseKind(k.String())
require.True(t, ok, k.String())
require.Equal(t, k, parsed)
}
require.Equal(t, "messages", KindMessages.String())
require.Equal(t, "bandwidth", KindBandwidth.String())
_, ok := ParseKind("nope")
require.False(t, ok)
}
func TestUsage_Count(t *testing.T) {
s := newTestService(t)
require.True(t, s.Allow("k", Rule{
Kind: KindMessages,
Daily: 10,
}, 3).OK)
require.True(t, s.Allow("k", Rule{
Kind: KindBandwidth,
Burst: 100,
Refill: time.Second,
}, 42).OK)
u := s.Usage("k")
require.Equal(t, int64(3), u.Count(KindMessages))
require.Equal(t, int64(42), u.Count(KindBandwidth))
require.Equal(t, int64(0), u.Count(KindEmails))
}
func TestLimiter_AllowNRefundsNegative(t *testing.T) {
s := newTestService(t)
l := s.Limiter("k", Rule{
Kind: KindBandwidth,
Burst: 10,
Refill: time.Hour,
})
require.True(t, l.AllowN(8))
require.False(t, l.AllowN(3))
require.Equal(t, int64(8), l.Value())
require.True(t, l.AllowN(-5)) // Refund, as the limit reader does when a later limiter rejects
require.True(t, l.AllowN(3))
require.Equal(t, int64(6), l.Value())
require.True(t, l.Allow())
require.Equal(t, int64(7), l.Value())
}
limit/util.go
+33
View File
@@ -0,0 +1,33 @@
package limit
import (
"time"
"heckel.io/ntfy/v2/util"
)
const (
fnvOffset = 2166136261 // FNV-1a 32-bit offset basis
fnvPrime = 16777619 // FNV-1a 32-bit prime
oneDay = 24 * time.Hour
)
// shardIndex returns the shard a key belongs to (FNV-1a, no allocation)
func shardIndex(key string) int {
h := uint32(fnvOffset)
for i := 0; i < len(key); i++ {
h ^= uint32(key[i])
h *= fnvPrime
}
return int(h % shardCount)
}
// untilReset returns how long until the next daily reset after now; a full day if now is the
// reset time itself
func untilReset(now time.Time, resetTime time.Time) time.Duration {
next := util.NextOccurrenceUTC(resetTime, now)
if !next.After(now) {
next = next.Add(oneDay)
}
return next.Sub(now)
}
limit/util_test.go
+31
View File
@@ -0,0 +1,31 @@
package limit
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestShardIndex(t *testing.T) {
seen := make(map[int]bool)
for _, key := range []string{"ip:1.2.3.4", "ip:1.2.3.5", "user:u_abc", "ip:2001:db8::", "a", "b", "c", "d", "e", "f"} {
i := shardIndex(key)
require.GreaterOrEqual(t, i, 0)
require.Less(t, i, shardCount)
require.Equal(t, i, shardIndex(key)) // Stable
seen[i] = true
}
require.Greater(t, len(seen), 3) // Spreads
}
func TestUntilReset(t *testing.T) {
resetTime := time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC) // Midnight UTC
now := time.Date(2026, 10, 25, 23, 59, 0, 0, time.UTC)
require.Equal(t, time.Minute, untilReset(now, resetTime))
now = time.Date(2026, 10, 25, 0, 0, 0, 0, time.UTC)
require.Equal(t, 24*time.Hour, untilReset(now, resetTime))
resetTime = time.Date(0, 0, 0, 6, 30, 0, 0, time.UTC)
now = time.Date(2026, 10, 25, 6, 0, 0, 0, time.UTC)
require.Equal(t, 30*time.Minute, untilReset(now, resetTime))
}
log/event.go
+2 -2
View File
@@ -209,9 +209,9 @@ func (e *Event) String() string {
}
func (e *Event) globalLevelWithOverride() Level {
mu.RLock()
mu.RLock() // Held while iterating: SetLevelOverride changes the overrides map in place
defer mu.RUnlock()
l, ov := level, overrides
mu.RUnlock()
if e.fields == nil {
return l
}
log/event_test.go
+23
View File
@@ -0,0 +1,23 @@
package log
import (
"fmt"
"testing"
)
func TestEvent_LevelOverridesChangedWhileLogging(t *testing.T) {
// Overrides can change while other goroutines log (log-level-overrides is hot-reloaded on
// SIGHUP); checking an event against them must not read the overrides map unlocked
t.Cleanup(ResetLevelOverrides)
done := make(chan struct{})
go func() {
defer close(done)
for i := 0; i < 1000; i++ {
Tag("mytag").Field("user_id", "u_123").Loggable(DebugLevel)
}
}()
for i := 0; i < 1000; i++ {
SetLevelOverride(fmt.Sprintf("field_%d", i), "value", DebugLevel)
}
<-done
}
mail/format.go
+22
View File
@@ -54,6 +54,12 @@ func formatMail(baseURL, senderIP, from, to string, m *model.Message) (string, e
}
trailer += fmt.Sprintf("Priority: %s", priority)
}
if m.Attachment != nil && m.Attachment.URL != "" {
if trailer != "" {
trailer += "\n"
}
trailer += "Attachment: " + formatAttachment(m.Attachment)
}
if trailer != "" {
message += "\n\n" + trailer
}
@@ -81,6 +87,22 @@ This message was sent by {ip} at {time} via {topicURL}`
return body, nil
}
// formatAttachment links the attachment rather than attaching it, since emails never expire
// but uploaded files do (and would turn ntfy into a file host)
func formatAttachment(a *model.Attachment) string {
details := make([]string, 0)
if a.Name != "" {
details = append(details, a.Name)
}
if a.Expires > 0 {
details = append(details, "expires "+time.Unix(a.Expires, 0).UTC().Format(time.RFC1123))
}
if len(details) == 0 {
return a.URL
}
return fmt.Sprintf("%s (%s)", a.URL, strings.Join(details, ", "))
}
func toEmojis(tags []string) (emojisOut []string, tagsOut []string) {
tagsOut = make([]string, 0)
emojisOut = make([]string, 0)
mail/format_test.go
+60
View File
@@ -147,3 +147,63 @@ Priority: max
This message was sent by 1.2.3.4 at Fri, 24 Dec 2021 21:43:24 UTC via https://ntfy.sh/alerts`
require.Equal(t, expected, actual)
}
func TestFormatMail_WithAttachment(t *testing.T) {
actual, _ := formatMail("https://ntfy.sh", "1.2.3.4", "ntfy@ntfy.sh", "phil@example.com", &model.Message{
ID: "abc",
Time: 1640382204,
Event: "message",
Topic: "alerts",
Priority: 4,
Message: "You received a file: backup.log",
Attachment: &model.Attachment{
Name: "backup.log",
Type: "text/plain; charset=utf-8",
Size: 1234,
Expires: 1640393004,
URL: "https://ntfy.sh/file/abc.txt",
},
})
expected := `From: "ntfy.sh/alerts" <ntfy@ntfy.sh>
To: phil@example.com
Date: Fri, 24 Dec 2021 21:43:24 +0000
Subject: You received a file: backup.log
Content-Type: text/plain; charset="utf-8"
You received a file: backup.log
Priority: high
Attachment: https://ntfy.sh/file/abc.txt (backup.log, expires Sat, 25 Dec 2021 00:43:24 UTC)
--
This message was sent by 1.2.3.4 at Fri, 24 Dec 2021 21:43:24 UTC via https://ntfy.sh/alerts`
require.Equal(t, expected, actual)
}
func TestFormatMail_WithExternalAttachment(t *testing.T) {
// Attachments passed via attach= are hosted elsewhere and never expire
actual, _ := formatMail("https://ntfy.sh", "1.2.3.4", "ntfy@ntfy.sh", "phil@example.com", &model.Message{
ID: "abc",
Time: 1640382204,
Event: "message",
Topic: "alerts",
Message: "Look at this",
Attachment: &model.Attachment{
Name: "cat.jpg",
URL: "https://example.com/cat.jpg",
},
})
expected := `From: "ntfy.sh/alerts" <ntfy@ntfy.sh>
To: phil@example.com
Date: Fri, 24 Dec 2021 21:43:24 +0000
Subject: Look at this
Content-Type: text/plain; charset="utf-8"
Look at this
Attachment: https://example.com/cat.jpg (cat.jpg)
--
This message was sent by 1.2.3.4 at Fri, 24 Dec 2021 21:43:24 UTC via https://ntfy.sh/alerts`
require.Equal(t, expected, actual)
}
mail/sender.go
+53 -1
View File
@@ -1,10 +1,12 @@
package mail
import (
"errors"
"fmt"
"mime"
"net"
"net/smtp"
"slices"
"strings"
"sync"
"time"
@@ -124,7 +126,7 @@ func (s *realSender) sendRaw(to string, message []byte) error {
}
var auth smtp.Auth
if s.config.SMTPUser != "" {
auth = smtp.PlainAuth("", s.config.SMTPUser, s.config.SMTPPass, host)
auth = newPlainOrLoginAuth(s.config.SMTPUser, s.config.SMTPPass, host)
}
return smtp.SendMail(s.config.SMTPAddr, auth, s.config.From, []string{to}, message)
}
@@ -138,3 +140,53 @@ func (s *realSender) count(ok bool) {
s.failure++
}
}
// plainOrLoginAuth uses PLAIN auth, or LOGIN auth if the server only supports that (e.g. OVH, Outlook).
// The stdlib net/smtp only implements PLAIN, so LOGIN is implemented here, see #930.
type plainOrLoginAuth struct {
plain smtp.Auth
username string
password string
login bool
step int
}
func newPlainOrLoginAuth(username, password, host string) smtp.Auth {
return &plainOrLoginAuth{
plain: smtp.PlainAuth("", username, password, host),
username: username,
password: password,
}
}
// Start lets PLAIN auth do the TLS and host name checks, so the password is never sent in the clear
func (a *plainOrLoginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
mechanism, resp, err := a.plain.Start(server)
if err != nil {
return "", nil, err
}
// Only switch to LOGIN if the server offers it but not PLAIN; otherwise behave like before
if !slices.Contains(server.Auth, "LOGIN") || slices.Contains(server.Auth, "PLAIN") {
return mechanism, resp, nil
}
a.login = true
return "LOGIN", nil, nil
}
// Next answers the LOGIN challenges in order (username, then password), since the prompt texts vary by server
func (a *plainOrLoginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
if !a.login {
return a.plain.Next(fromServer, more)
} else if !more {
return nil, nil
}
a.step++
switch a.step {
case 1:
return []byte(a.username), nil
case 2:
return []byte(a.password), nil
default:
return nil, errors.New("unexpected server challenge")
}
}
mail/sender_test.go
+261
View File
@@ -0,0 +1,261 @@
package mail
import (
"bufio"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"fmt"
"math/big"
"net"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestSender_AuthLogin(t *testing.T) {
smtpServer := newTestSMTPServer(t, "LOGIN")
sender := NewSender(smtpServer.config("phil", "secret"))
require.Nil(t, sender.SendEmailVerification("someone@example.com", "https://ntfy.example.com/verify"))
require.Equal(t, "LOGIN", smtpServer.mechanism())
require.Contains(t, smtpServer.data(), "https://ntfy.example.com/verify")
}
func TestSender_AuthLogin_WrongPassword(t *testing.T) {
smtpServer := newTestSMTPServer(t, "LOGIN")
sender := NewSender(smtpServer.config("phil", "wrong"))
require.Error(t, sender.SendEmailVerification("someone@example.com", "https://ntfy.example.com/verify"))
require.Empty(t, smtpServer.data())
}
func TestSender_AuthLogin_RefusesUnencryptedRemoteHost(t *testing.T) {
// 127.0.0.2 is not "localhost" to net/smtp, so without TLS it must not send the password
smtpServer := newTestSMTPServerOn(t, "127.0.0.2:0", "LOGIN")
sender := NewSender(smtpServer.config("phil", "secret"))
err := sender.SendEmailVerification("someone@example.com", "https://ntfy.example.com/verify")
require.ErrorContains(t, err, "unencrypted connection")
require.Empty(t, smtpServer.mechanism())
}
func TestSender_AuthLogin_StartTLS(t *testing.T) {
// LOGIN is only offered after STARTTLS (like Outlook), on a non-localhost address, so the
// password can only be sent because the connection is encrypted
if runtime.GOOS != "linux" && runtime.GOOS != "freebsd" {
t.Skip("trusting the test certificate via SSL_CERT_FILE only works on Linux/BSD")
}
smtpServer := newTestSMTPServerOn(t, "127.0.0.2:0", "LOGIN")
smtpServer.tlsConfig = newTestTLSConfig(t, "127.0.0.2")
sender := NewSender(smtpServer.config("phil", "secret"))
require.Nil(t, sender.SendEmailVerification("someone@example.com", "https://ntfy.example.com/verify"))
require.Equal(t, "LOGIN", smtpServer.mechanism())
require.True(t, smtpServer.authenticatedOverTLS())
require.Contains(t, smtpServer.data(), "https://ntfy.example.com/verify")
}
func TestSender_AuthPlainPreferredOverLogin(t *testing.T) {
smtpServer := newTestSMTPServer(t, "LOGIN PLAIN")
sender := NewSender(smtpServer.config("phil", "secret"))
require.Nil(t, sender.SendEmailVerification("someone@example.com", "https://ntfy.example.com/verify"))
require.Equal(t, "PLAIN", smtpServer.mechanism())
}
func TestSender_AuthNoSupportedMechanism(t *testing.T) {
smtpServer := newTestSMTPServer(t, "CRAM-MD5")
sender := NewSender(smtpServer.config("phil", "secret"))
require.Error(t, sender.SendEmailVerification("someone@example.com", "https://ntfy.example.com/verify"))
require.Empty(t, smtpServer.data())
}
// testSMTPServer is a minimal SMTP server that only supports the given AUTH mechanisms
// and accepts the user "phil" with password "secret". If tlsConfig is set, it offers
// STARTTLS and only advertises AUTH after the connection is encrypted.
type testSMTPServer struct {
listener net.Listener
mechanisms string
tlsConfig *tls.Config
usedMech string
usedTLS bool
received string
mu sync.Mutex
}
func newTestSMTPServer(t *testing.T, mechanisms string) *testSMTPServer {
return newTestSMTPServerOn(t, "127.0.0.1:0", mechanisms)
}
func newTestSMTPServerOn(t *testing.T, addr, mechanisms string) *testSMTPServer {
listener, err := net.Listen("tcp", addr)
if err != nil {
t.Skipf("cannot listen on %s: %s", addr, err.Error())
}
s := &testSMTPServer{listener: listener, mechanisms: mechanisms}
t.Cleanup(func() {
listener.Close()
})
go s.serve()
return s
}
func (s *testSMTPServer) config(user, pass string) *Config {
return &Config{
BaseURL: "https://ntfy.example.com",
SMTPAddr: s.listener.Addr().String(),
SMTPUser: user,
SMTPPass: pass,
From: "ntfy@example.com",
}
}
func (s *testSMTPServer) mechanism() string {
s.mu.Lock()
defer s.mu.Unlock()
return s.usedMech
}
func (s *testSMTPServer) authenticatedOverTLS() bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.usedTLS
}
func (s *testSMTPServer) data() string {
s.mu.Lock()
defer s.mu.Unlock()
return s.received
}
func (s *testSMTPServer) serve() {
for {
conn, err := s.listener.Accept()
if err != nil {
return
}
go s.handle(conn)
}
}
func (s *testSMTPServer) handle(conn net.Conn) {
defer func() {
conn.Close()
}()
r := bufio.NewReader(conn)
encrypted := false
reply := func(format string, args ...any) {
fmt.Fprintf(conn, format+"\r\n", args...)
}
readLine := func() string {
line, _ := r.ReadString('\n')
return strings.TrimRight(line, "\r\n")
}
decode := func(s string) string {
b, _ := base64.StdEncoding.DecodeString(s)
return string(b)
}
authenticated := false
reply("220 localhost ESMTP test")
for {
line := readLine()
cmd := strings.ToUpper(line)
switch {
case line == "":
return
case strings.HasPrefix(cmd, "EHLO") && s.tlsConfig != nil && !encrypted:
reply("250-localhost")
reply("250 STARTTLS")
case strings.HasPrefix(cmd, "EHLO"):
reply("250-localhost")
reply("250 AUTH %s", s.mechanisms)
case strings.HasPrefix(cmd, "STARTTLS") && s.tlsConfig != nil && !encrypted:
reply("220 Ready to start TLS")
tlsConn := tls.Server(conn, s.tlsConfig)
if err := tlsConn.Handshake(); err != nil {
return
}
conn, r, encrypted = tlsConn, bufio.NewReader(tlsConn), true
case strings.HasPrefix(cmd, "AUTH LOGIN") && strings.Contains(s.mechanisms, "LOGIN"):
reply("334 %s", base64.StdEncoding.EncodeToString([]byte("Username:")))
user := decode(readLine())
reply("334 %s", base64.StdEncoding.EncodeToString([]byte("Password:")))
pass := decode(readLine())
authenticated = s.authenticate("LOGIN", user, pass, encrypted, reply)
case strings.HasPrefix(cmd, "AUTH PLAIN ") && strings.Contains(s.mechanisms, "PLAIN"):
parts := strings.Split(decode(line[len("AUTH PLAIN "):]), "\x00")
authenticated = len(parts) == 3 && s.authenticate("PLAIN", parts[1], parts[2], encrypted, reply)
case strings.HasPrefix(cmd, "AUTH"):
reply("504 Unrecognized authentication type")
case strings.HasPrefix(cmd, "MAIL") && !authenticated:
reply("530 Authentication required")
case strings.HasPrefix(cmd, "MAIL"), strings.HasPrefix(cmd, "RCPT"), strings.HasPrefix(cmd, "RSET"):
reply("250 OK")
case strings.HasPrefix(cmd, "DATA"):
reply("354 Go ahead")
var data strings.Builder
for l := readLine(); l != "."; l = readLine() {
data.WriteString(l + "\n")
}
s.mu.Lock()
s.received = data.String()
s.mu.Unlock()
reply("250 OK")
case strings.HasPrefix(cmd, "QUIT"):
reply("221 Bye")
return
default:
reply("502 Command not implemented")
}
}
}
func (s *testSMTPServer) authenticate(mechanism, user, pass string, encrypted bool, reply func(format string, args ...any)) bool {
if user != "phil" || pass != "secret" {
reply("535 Authentication failed")
return false
}
s.mu.Lock()
s.usedMech = mechanism
s.usedTLS = encrypted
s.mu.Unlock()
reply("235 Authentication successful")
return true
}
// newTestTLSConfig creates a self-signed certificate for the given IP and makes the Go client trust it
// via SSL_CERT_FILE, since net/smtp.SendMail does not accept a custom TLS config. The system cert pool
// is loaded once per process (during the first handshake), so all tests share one certificate.
func newTestTLSConfig(t *testing.T, ip string) *tls.Config {
testTLSOnce.Do(func() {
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
require.Nil(t, err)
template := &x509.Certificate{
SerialNumber: big.NewInt(1),
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
IPAddresses: []net.IP{net.ParseIP(ip)},
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
require.Nil(t, err)
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
certFile := filepath.Join(t.TempDir(), "cert.pem")
require.Nil(t, os.WriteFile(certFile, certPEM, 0600))
require.Nil(t, os.Setenv("SSL_CERT_FILE", certFile))
testTLSCert = tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
})
return &tls.Config{Certificates: []tls.Certificate{testTLSCert}}
}
var (
testTLSOnce sync.Once
testTLSCert tls.Certificate
)
main.go
+1
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"runtime"
_ "time/tzdata" // Named time zones for delayed messages must work without system zone files (Alpine ARM images)
"github.com/urfave/cli/v2"
"heckel.io/ntfy/v2/cmd"
message/cache.go
+271 -83
View File
@@ -22,23 +22,51 @@ const (
// NoLimit reads a topic's cached messages without a size budget.
NoLimit = 0
// closeFlushTimeout bounds how long Close waits for the last batch to be written. A batch
// write takes milliseconds, so this only ever trips when the database is wedged, and then
// shutdown must still finish well inside systemd's stop timeout.
closeFlushTimeout = 5 * time.Second
claimTimeout = 2 * time.Minute // How long a claim hides a due message from other nodes; must exceed dispatching one claim
claimBatchSize = 1000 // Rows per claim, so the dispatch time that claimTimeout must cover is bounded
// queueBufferedBatches is how many full batches can wait for the batch writer before publishes
// block. 100 batches of the default 100 messages are ~10 MB typical, ~60 MB at the 4 KB limit.
// Every buffered batch must be written within closeFlushTimeout on shutdown.
queueBufferedBatches = 100
// insertMessageColumns is the number of values insertMessageArgs returns per message, which is
// the number of columns in each backend's INSERT statement
insertMessageColumns = 24
)
var errNoRows = errors.New("no rows found")
const (
recentReplayWindow = time.Minute // A time-based replay from this close to now reads the primary (see messagesSinceTime)
)
var (
errNoRows = errors.New("no rows found")
errSinceIDNeedsMarker = errors.New("since=<id> needs ResolveMarker and MessagesAfterMarker")
)
// queries holds the database-specific SQL queries
type queries struct {
insertMessage string
insertMessages func(tx *sql.Tx, ms []*model.Message) error // Writes a batch of messages in as few statements as possible
selectScheduledMessageIDsBySeqID string
deleteScheduledBySequenceID string
updateMessagesForTopicExpiry string
selectMessagesByID string
selectMessagesSinceTime string
selectMessagesSinceTimeScheduled string
selectMessageRowID string
selectMessagesSinceID string
selectMessagesSinceIDScheduled string
selectRecentRowIDs string
selectMessagesAfterRowID string
selectMessagesLatest string
selectMessagesDue string
claimMessagesDue string // Postgres-only: claims due rows and stamps claimed_at; empty for SQLite/mem
deleteExpiredMessages string
updateMessagePublished string
selectMessagesCount string
@@ -56,22 +84,36 @@ type queries struct {
type Cache struct {
db *db.DB
queue *util.BatchingQueue[*model.Message]
written chan struct{} // Closed once the batch writer has drained the queue (after queue.Close)
nop bool
mu *sync.Mutex // nil for PostgreSQL (concurrent writes supported), set for SQLite (single writer)
queries queries
// claimTimeout is how long this node's claim on a due message holds off the others; it is a
// field only so tests can shorten it
claimTimeout time.Duration
}
// Marker is a resolved since=<mid> marker (see ResolveMarker): its row id and publish time, and
// the database that knew it, which the replay is read from
type Marker struct {
rowID int64
time int64
db *sql.DB // The primary, if the replica did not know the marker yet: it lacks the newest rows too
}
func newCache(db *db.DB, queries queries, mu *sync.Mutex, batchSize int, batchTimeout time.Duration, nop bool) *Cache {
var queue *util.BatchingQueue[*model.Message]
if batchSize > 0 || batchTimeout > 0 {
queue = util.NewBatchingQueue[*model.Message](batchSize, batchTimeout)
queue = util.NewBatchingQueue[*model.Message](batchSize, batchTimeout, queueBufferedBatches)
}
c := &Cache{
db: db,
queue: queue,
nop: nop,
mu: mu,
queries: queries,
db: db,
queue: queue,
written: make(chan struct{}),
nop: nop,
mu: mu,
queries: queries,
claimTimeout: claimTimeout,
}
go c.processMessageBatches()
return c
@@ -119,69 +161,9 @@ func (c *Cache) addMessages(ms []*model.Message) error {
return err
}
defer tx.Rollback()
stmt, err := tx.Prepare(c.queries.insertMessage)
if err != nil {
if err := c.queries.insertMessages(tx, ms); err != nil {
return err
}
defer stmt.Close()
for _, m := range ms {
if m.Event != model.MessageEvent && m.Event != model.MessageDeleteEvent && m.Event != model.MessageClearEvent {
return model.ErrUnexpectedMessageType
}
published := m.Time <= time.Now().Unix()
tags := util.SanitizeUTF8(strings.Join(m.Tags, ","))
var attachmentName, attachmentType, attachmentURL string
var attachmentSize, attachmentExpires int64
var attachmentDeleted bool
if m.Attachment != nil {
attachmentName = util.SanitizeUTF8(m.Attachment.Name)
attachmentType = util.SanitizeUTF8(m.Attachment.Type)
attachmentSize = m.Attachment.Size
attachmentExpires = m.Attachment.Expires
attachmentURL = util.SanitizeUTF8(m.Attachment.URL)
}
var actionsStr string
if len(m.Actions) > 0 {
actionsBytes, err := json.Marshal(m.Actions)
if err != nil {
return err
}
actionsStr = string(actionsBytes)
}
var sender string
if m.Sender.IsValid() {
sender = m.Sender.String()
}
_, err := stmt.Exec(
m.ID,
m.SequenceID,
m.Time,
m.Event,
m.Expires,
util.SanitizeUTF8(m.Topic),
util.SanitizeUTF8(m.Message),
util.SanitizeUTF8(m.Title),
m.Priority,
tags,
util.SanitizeUTF8(m.Click),
util.SanitizeUTF8(m.Icon),
actionsStr,
attachmentName,
attachmentType,
attachmentSize,
attachmentExpires,
attachmentURL,
attachmentDeleted, // Always zero
sender,
m.User,
util.SanitizeUTF8(m.ContentType),
m.Encoding,
published,
)
if err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
log.Tag(tagMessageCache).Err(err).Error("Writing %d message(s) failed (took %v)", len(ms), time.Since(start))
return err
@@ -190,6 +172,64 @@ func (c *Cache) addMessages(ms []*model.Message) error {
return nil
}
// insertMessageArgs returns the values of one message row, in the order of the columns in the
// backends' INSERT statements (postgresInsertMessagesQuery, sqliteInsertMessagesQuery)
func insertMessageArgs(m *model.Message) ([]any, error) {
if m.Event != model.MessageEvent && m.Event != model.MessageDeleteEvent && m.Event != model.MessageClearEvent {
return nil, model.ErrUnexpectedMessageType
}
published := m.Time <= time.Now().Unix()
tags := util.SanitizeUTF8(strings.Join(m.Tags, ","))
var attachmentName, attachmentType, attachmentURL string
var attachmentSize, attachmentExpires int64
var attachmentDeleted bool
if m.Attachment != nil {
attachmentName = util.SanitizeUTF8(m.Attachment.Name)
attachmentType = util.SanitizeUTF8(m.Attachment.Type)
attachmentSize = m.Attachment.Size
attachmentExpires = m.Attachment.Expires
attachmentURL = util.SanitizeUTF8(m.Attachment.URL)
}
var actionsStr string
if len(m.Actions) > 0 {
actionsBytes, err := json.Marshal(m.Actions)
if err != nil {
return nil, err
}
actionsStr = string(actionsBytes)
}
var sender string
if m.Sender.IsValid() {
sender = m.Sender.String()
}
return []any{
m.ID,
m.SequenceID,
m.Time,
m.Event,
m.Expires,
util.SanitizeUTF8(m.Topic),
util.SanitizeUTF8(m.Message),
util.SanitizeUTF8(m.Title),
m.Priority,
tags,
util.SanitizeUTF8(m.Click),
util.SanitizeUTF8(m.Icon),
actionsStr,
attachmentName,
attachmentType,
attachmentSize,
attachmentExpires,
attachmentURL,
attachmentDeleted, // Always zero
sender,
m.User,
util.SanitizeUTF8(m.ContentType),
m.Encoding,
published,
}, nil
}
// Messages returns all cached messages for a topic, oldest first. Prefer MessagesCapped on
// request paths: an uncapped replay of a busy topic is as large as the topic's entire cache.
func (c *Cache) Messages(topic string, since model.SinceMarker, scheduled bool) ([]*model.Message, error) {
@@ -199,7 +239,8 @@ func (c *Cache) Messages(topic string, since model.SinceMarker, scheduled bool)
// MessagesCapped returns cached messages for a topic, oldest first, keeping the newest messages
// that fit in maxBytes worth of Message.Size (0 = no budget). The bool reports whether older messages
// were dropped, so the caller can tell the client that what it got is incomplete.
// were dropped, so the caller can tell the client that what it got is incomplete. A since=<id>
// marker is rejected: it goes through ResolveMarker and MessagesAfterMarker instead.
func (c *Cache) MessagesCapped(topic string, since model.SinceMarker, scheduled bool, maxBytes int64) ([]*model.Message, bool, error) {
if since.IsNone() {
return make([]*model.Message, 0), false, nil
@@ -207,15 +248,21 @@ func (c *Cache) MessagesCapped(topic string, since model.SinceMarker, scheduled
messages, err := c.messagesLatest(topic)
return messages, false, err
} else if since.IsID() {
return c.messagesSinceID(topic, since, scheduled, maxBytes)
return nil, false, errSinceIDNeedsMarker
}
return c.messagesSinceTime(topic, since, scheduled, maxBytes)
}
// messagesSinceTime reads a time-based replay. A recent one (the fallback for a since=<id>
// marker the cache does not know yet) reads the primary: what a lagging replica lacks went out
// live before the client arrived, so the tail does not resend it either.
func (c *Cache) messagesSinceTime(topic string, since model.SinceMarker, scheduled bool, maxBytes int64) ([]*model.Message, bool, error) {
var rows *sql.Rows
var err error
rdb := c.db.ReadOnly()
if time.Since(since.Time()) < recentReplayWindow {
rdb = c.db.Primary()
}
if scheduled {
rows, err = rdb.Query(c.queries.selectMessagesSinceTimeScheduled, topic, since.Time().Unix())
} else {
@@ -227,14 +274,38 @@ func (c *Cache) messagesSinceTime(topic string, since model.SinceMarker, schedul
return readMessagesCapped(rows, maxBytes)
}
func (c *Cache) messagesSinceID(topic string, since model.SinceMarker, scheduled bool, maxBytes int64) ([]*model.Message, bool, error) {
// ResolveMarker looks up a since=<mid> marker once, for replaying any number of topics after it
// (see MessagesAfterMarker); nil if the cache does not know it (not written yet, or expired)
func (c *Cache) ResolveMarker(mid string) (*Marker, error) {
rowID, markerTime, rdb, err := c.resolveMarker(mid)
if err != nil || rowID == 0 {
return nil, err
}
return &Marker{rowID: rowID, time: markerTime, db: rdb}, nil
}
// MessagesAfterMarker is MessagesCapped for a resolved since=<mid> marker. The query matches
// "id > marker row id OR (time >= marker time AND id <> marker row id)":
// - id > marker row id: the rows written after the marker, the usual case
// - time >= marker time: rows published in the marker's second or later but written before it.
// Nodes write their batches independently, and late when one is cut off from the database,
// so the marker itself may have been written late, after messages published later than it.
// - id <> marker row id: the marker matches its own time, and the client has it
//
// Example: the client's marker M was published at 12:00:05 on a node cut off from the database
// and written when it came back, as row 900; N was published at 12:00:07 on another node and
// written right away, as row 850. "id > 900" misses N, "time >= 12:00:05" finds it.
//
// The cost is a duplicate for each message published in the marker's second before it; clients
// drop duplicates by id. Times are whole seconds on purpose: they come from each node's clock,
// and a second hides clock skew between nodes that milliseconds would turn into losses.
func (c *Cache) MessagesAfterMarker(topic string, marker *Marker, scheduled bool, maxBytes int64) ([]*model.Message, bool, error) {
var rows *sql.Rows
var err error
rdb := c.db.ReadOnly()
if scheduled {
rows, err = rdb.Query(c.queries.selectMessagesSinceIDScheduled, topic, since.ID())
rows, err = marker.db.Query(c.queries.selectMessagesSinceIDScheduled, topic, marker.rowID, marker.time)
} else {
rows, err = rdb.Query(c.queries.selectMessagesSinceID, topic, since.ID())
rows, err = marker.db.Query(c.queries.selectMessagesSinceID, topic, marker.rowID, marker.time)
}
if err != nil {
return nil, false, err
@@ -242,6 +313,38 @@ func (c *Cache) messagesSinceID(topic string, since model.SinceMarker, scheduled
return readMessagesCapped(rows, maxBytes)
}
// resolveMarker resolves a since=<mid> marker to its row id and time, the replay cut-offs, and
// the database handle to read the replay from. It asks the replica first and, only if the mid is
// unknown there, the primary: a client that reconnects right after receiving a message can be
// ahead of replication. The replay is then read from the primary as well: what the replica lacks
// went out live before the client arrived, so the tail does not resend it either. A marker
// unknown to both (not written yet, or expired) resolves to 0, 0, which replays the topic's
// entire retained history; the server avoids that for markers it saw go out (sinceForTopic).
func (c *Cache) resolveMarker(mid string) (int64, int64, *sql.DB, error) {
rdb := c.db.ReadOnly()
rowID, markerTime, err := c.resolveMarkerOn(rdb, mid)
if err != nil {
return 0, 0, rdb, err
}
if rowID == 0 && rdb != c.db.Primary() {
rowID, markerTime, err = c.resolveMarkerOn(c.db.Primary(), mid)
return rowID, markerTime, c.db.Primary(), err
}
return rowID, markerTime, rdb, nil
}
// resolveMarkerOn looks the mid up on one database handle; 0 means unknown
func (c *Cache) resolveMarkerOn(h *sql.DB, mid string) (int64, int64, error) {
var rowID, markerTime int64
err := h.QueryRow(c.queries.selectMessageRowID, mid).Scan(&rowID, &markerTime)
if errors.Is(err, sql.ErrNoRows) {
return 0, 0, nil
} else if err != nil {
return 0, 0, err
}
return rowID, markerTime, nil
}
func (c *Cache) messagesLatest(topic string) ([]*model.Message, error) {
rows, err := c.db.ReadOnly().Query(c.queries.selectMessagesLatest, topic)
if err != nil {
@@ -250,8 +353,12 @@ func (c *Cache) messagesLatest(topic string) ([]*model.Message, error) {
return readMessages(rows)
}
// MessagesDue returns all messages that are due for publishing
// MessagesDue returns all messages that are due for publishing. On Postgres it CLAIMS them, so
// every node may call it, leader or not: each row is handed to exactly one caller.
func (c *Cache) MessagesDue() ([]*model.Message, error) {
if c.queries.claimMessagesDue != "" {
return c.claimMessagesDue()
}
rows, err := c.db.Query(c.queries.selectMessagesDue, time.Now().Unix())
if err != nil {
return nil, err
@@ -259,6 +366,23 @@ func (c *Cache) MessagesDue() ([]*model.Message, error) {
return readMessages(rows)
}
// claimMessagesDue stamps the due rows it takes, so concurrent senders on other nodes get
// disjoint sets. The rows stay unpublished: MarkPublished still runs after delivery.
func (c *Cache) claimMessagesDue() ([]*model.Message, error) {
now := time.Now().Unix()
rows, err := c.db.Query(c.queries.claimMessagesDue, now, now-int64(c.claimTimeout.Seconds()), claimBatchSize, now)
if err != nil {
return nil, err
}
return readMessages(rows) // Reads all rows and closes them
}
// SetClaimTimeoutForTest shortens the claim hold-off, so a test can see an abandoned claim
// become claimable again without waiting out the real timeout.
func (c *Cache) SetClaimTimeoutForTest(timeout time.Duration) {
c.claimTimeout = timeout
}
// DeleteExpiredMessages deletes up to `limit` expired messages in a single query
// and returns the number of deleted rows.
func (c *Cache) DeleteExpiredMessages(limit int) (int64, error) {
@@ -284,6 +408,39 @@ func (c *Cache) Message(id string) (*model.Message, error) {
return readMessage(rows)
}
// Tail returns a tail that starts after the last message written so far. The gaps between the
// last tailPageSize row ids start out as holes: a row id handed out but not committed yet, below
// one that already is, must not be skipped.
//
// Example: the last row ids are 108, 106, 105 and 103. Walking them oldest first, 103 only sets
// the cursor (the gap below the oldest is unknown), 105 adds hole 104, 106 adds nothing, and 108
// adds hole 107. The tail starts at 108 with holes {104, 107}: if 107 was another node's write
// that commits a moment later, the first read returns it; if 104 was rolled back, it never shows
// up and the hole expires after tailHoleTimeout.
func (c *Cache) Tail() (*Tail, error) {
rows, err := c.db.Primary().Query(c.queries.selectRecentRowIDs, tailPageSize)
if err != nil {
return nil, err
}
rowIDs, err := readRowIDs(rows)
if err != nil {
return nil, err
}
t := &Tail{
cache: c,
holes: make(map[int64]time.Time),
done: make(chan struct{}),
}
now := time.Now()
for _, rowID := range slices.Backward(rowIDs) {
if t.cursor > 0 {
t.addHoles(t.cursor, rowID, now)
}
t.cursor = rowID
}
return t, nil
}
// UpdateMessageTime updates the time column for a message by ID. This is only used for testing.
func (c *Cache) UpdateMessageTime(messageID string, timestamp int64) error {
c.maybeLock()
@@ -433,11 +590,12 @@ func (c *Cache) readAttachmentBytesUsed(rows *sql.Rows) (int64, error) {
return size, nil
}
// UpdateStats updates the total message count statistic
func (c *Cache) UpdateStats(messages int64) error {
// AddStats adds the given delta to the total message count statistic. Increments (rather than
// absolute writes) let multiple cluster nodes contribute to one shared counter.
func (c *Cache) AddStats(delta int64) error {
c.maybeLock()
defer c.maybeUnlock()
_, err := c.db.Exec(c.queries.updateStats, messages)
_, err := c.db.Exec(c.queries.updateStats, delta)
return err
}
@@ -458,11 +616,27 @@ func (c *Cache) Stats() (messages int64, err error) {
}
// Close closes the underlying database connection
// Close writes the messages still waiting in the batch queue, then closes the database. A
// database that does not accept the last batch delays shutdown by at most closeFlushTimeout.
func (c *Cache) Close() error {
if c.queue != nil {
flushed := make(chan struct{})
go func() {
c.queue.Close() // Hands the pending messages to the batch writer
<-c.written
close(flushed)
}()
select {
case <-flushed:
case <-time.After(closeFlushTimeout):
log.Tag(tagMessageCache).Warn("Giving up on the last message batch after %v", closeFlushTimeout)
}
}
return c.db.Close()
}
func (c *Cache) processMessageBatches() {
defer close(c.written)
if c.queue == nil {
return
}
@@ -525,11 +699,12 @@ func readMessages(rows *sql.Rows) ([]*model.Message, error) {
return messages, nil
}
func readMessage(rows *sql.Rows) (*model.Message, error) {
// readMessage reads one message row; extra receives the columns selected after the message columns
func readMessage(rows *sql.Rows, extra ...any) (*model.Message, error) {
var timestamp, expires, attachmentSize, attachmentExpires int64
var priority int
var id, sequenceID, event, topic, msg, title, tagsStr, click, icon, actionsStr, attachmentName, attachmentType, attachmentURL, sender, user, contentType, encoding string
err := rows.Scan(
dest := []any{
&id,
&sequenceID,
&timestamp,
@@ -552,8 +727,8 @@ func readMessage(rows *sql.Rows) (*model.Message, error) {
&user,
&contentType,
&encoding,
)
if err != nil {
}
if err := rows.Scan(append(dest, extra...)...); err != nil {
return nil, err
}
var tags []string
@@ -602,6 +777,19 @@ func readMessage(rows *sql.Rows) (*model.Message, error) {
}, nil
}
func readRowIDs(rows *sql.Rows) ([]int64, error) {
defer rows.Close()
rowIDs := make([]int64, 0)
for rows.Next() {
var rowID int64
if err := rows.Scan(&rowID); err != nil {
return nil, err
}
rowIDs = append(rowIDs, rowID)
}
return rowIDs, rows.Err()
}
func readStrings(rows *sql.Rows) ([]string, error) {
strs := make([]string, 0)
for rows.Next() {
message/cache_postgres.go
+85 -13
View File
@@ -1,17 +1,23 @@
package message
import (
"database/sql"
"time"
"heckel.io/ntfy/v2/db"
"heckel.io/ntfy/v2/db/schema"
"heckel.io/ntfy/v2/model"
)
// PostgreSQL runtime query constants
const (
postgresInsertMessageQuery = `
// postgresInsertMessagesQuery writes a whole batch in one statement with one array parameter
// per column, in the order of insertMessageArgs. The statement text is the same for every batch
// size: pgx prepares each distinct text server-side and keeps up to 512 per connection, and a
// multi-row INSERT per batch size left hundreds of multi-MB plans in every pooled backend.
postgresInsertMessagesQuery = `
INSERT INTO message (mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_deleted, sender, user_id, content_type, encoding, published)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24)
SELECT * FROM unnest($1::TEXT[], $2::TEXT[], $3::BIGINT[], $4::TEXT[], $5::BIGINT[], $6::TEXT[], $7::TEXT[], $8::TEXT[], $9::INT[], $10::TEXT[], $11::TEXT[], $12::TEXT[], $13::TEXT[], $14::TEXT[], $15::TEXT[], $16::BIGINT[], $17::BIGINT[], $18::TEXT[], $19::BOOLEAN[], $20::TEXT[], $21::TEXT[], $22::TEXT[], $23::TEXT[], $24::BOOLEAN[])
`
postgresSelectScheduledMessageIDsBySeqIDQuery = `SELECT mid FROM message WHERE topic = $1 AND sequence_id = $2 AND published = FALSE`
postgresDeleteScheduledBySequenceIDQuery = `DELETE FROM message WHERE topic = $1 AND sequence_id = $2 AND published = FALSE`
@@ -33,21 +39,47 @@ const (
WHERE topic = $1 AND time >= $2
ORDER BY time DESC, id DESC
`
postgresSelectMessageRowIDQuery = `SELECT id, time FROM message WHERE mid = $1 LIMIT 1`
// postgresSelectMessagesSinceIDQuery replays after a since=<mid> marker (see Cache.MessagesAfterMarker):
// rows published in the marker's second ($3) or later, except the marker ($2), and rows written after
// the marker but published before it. Each part starts its index scan at the marker; as one OR, every
// row of the topic was read (incident 2026-10-16). "time + 0" keeps the second part off the time index,
// whose scan would start at the topic's oldest row.
postgresSelectMessagesSinceIDQuery = `
SELECT mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user_id, content_type, encoding
FROM message
WHERE topic = $1
AND id > COALESCE((SELECT id FROM message WHERE mid = $2), 0)
AND published = TRUE
FROM (
SELECT * FROM message WHERE topic = $1 AND published = TRUE AND time >= $3 AND id <> $2
UNION ALL
SELECT * FROM message WHERE topic = $1 AND published = TRUE AND id > $2 AND time + 0 < $3
) m
ORDER BY time DESC, id DESC
`
postgresSelectMessagesSinceIDIncludeScheduledQuery = `
SELECT mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user_id, content_type, encoding
FROM message
WHERE topic = $1
AND (id > COALESCE((SELECT id FROM message WHERE mid = $2), 0) OR published = FALSE)
FROM (
SELECT * FROM message WHERE topic = $1 AND published = TRUE AND time >= $3 AND id <> $2
UNION ALL
SELECT * FROM message WHERE topic = $1 AND published = TRUE AND id > $2 AND time + 0 < $3
UNION ALL
SELECT * FROM message WHERE topic = $1 AND published = FALSE
) m
ORDER BY time DESC, id DESC
`
postgresSelectRecentRowIDsQuery = `SELECT id FROM message ORDER BY id DESC LIMIT $1`
// postgresSelectMessagesAfterRowIDQuery reads rows in write order: those after the cursor ($1),
// plus the skipped row ids ($2, a JSON array) that were not committed yet when the cursor passed.
// The parts are separate so that the primary key checks every condition, also in the plan made
// without the parameter values; as one OR, that plan read the whole table.
postgresSelectMessagesAfterRowIDQuery = `
SELECT mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user_id, content_type, encoding, id, published
FROM (
SELECT * FROM message WHERE id > $1
UNION ALL
SELECT * FROM message WHERE id = ANY(ARRAY(SELECT jsonb_array_elements_text($2::JSONB)::BIGINT)) AND id <= $1
) m
ORDER BY id
LIMIT $3
`
postgresSelectMessagesLatestQuery = `
SELECT mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user_id, content_type, encoding
FROM message
@@ -76,23 +108,42 @@ const (
postgresSelectAttachmentsSizeByUserIDQuery = `SELECT COALESCE(SUM(attachment_size), 0) FROM message WHERE user_id = $1 AND attachment_expires >= $2`
postgresSelectAttachmentsWithSizesQuery = `SELECT mid, attachment_size FROM message WHERE attachment_expires > $1 AND attachment_deleted = FALSE`
postgresSelectStatsQuery = `SELECT value FROM message_stats WHERE key = 'messages'`
postgresUpdateStatsQuery = `UPDATE message_stats SET value = $1 WHERE key = 'messages'`
postgresSelectStatsQuery = `SELECT value FROM message_stats WHERE key = 'messages'`
// postgresClaimMessagesDueQuery claims due rows in one statement: the inner SELECT locks what
// it picks and skips what another node is claiming, so concurrent senders get disjoint sets.
// Unclaimed rows have claimed_at 0, so the cutoff ($2) matches them too.
postgresClaimMessagesDueQuery = `
WITH due AS (
SELECT id FROM message
WHERE time <= $1 AND published = FALSE AND claimed_at <= $2
ORDER BY time, id
LIMIT $3
FOR UPDATE SKIP LOCKED
)
UPDATE message SET claimed_at = $4 WHERE id IN (SELECT id FROM due)
RETURNING mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user_id, content_type, encoding
`
postgresUpdateStatsQuery = `UPDATE message_stats SET value = value + $1 WHERE key = 'messages'`
postgresUpdateMessageTimeQuery = `UPDATE message SET time = $1 WHERE mid = $2`
)
var postgresQueries = queries{
insertMessage: postgresInsertMessageQuery,
insertMessages: postgresInsertMessages,
selectScheduledMessageIDsBySeqID: postgresSelectScheduledMessageIDsBySeqIDQuery,
deleteScheduledBySequenceID: postgresDeleteScheduledBySequenceIDQuery,
updateMessagesForTopicExpiry: postgresUpdateMessagesForTopicExpiryQuery,
selectMessagesByID: postgresSelectMessagesByIDQuery,
selectMessagesSinceTime: postgresSelectMessagesSinceTimeQuery,
selectMessagesSinceTimeScheduled: postgresSelectMessagesSinceTimeIncludeScheduledQuery,
selectMessageRowID: postgresSelectMessageRowIDQuery,
selectMessagesSinceID: postgresSelectMessagesSinceIDQuery,
selectMessagesSinceIDScheduled: postgresSelectMessagesSinceIDIncludeScheduledQuery,
selectRecentRowIDs: postgresSelectRecentRowIDsQuery,
selectMessagesAfterRowID: postgresSelectMessagesAfterRowIDQuery,
selectMessagesLatest: postgresSelectMessagesLatestQuery,
selectMessagesDue: postgresSelectMessagesDueQuery,
claimMessagesDue: postgresClaimMessagesDueQuery,
deleteExpiredMessages: postgresDeleteExpiredMessagesQuery,
updateMessagePublished: postgresUpdateMessagePublishedQuery,
selectMessagesCount: postgresSelectMessagesCountQuery,
@@ -108,8 +159,29 @@ var postgresQueries = queries{
// NewPostgresStore creates a new PostgreSQL-backed message cache store using an existing database connection pool.
func NewPostgresStore(d *db.DB, batchSize int, batchTimeout time.Duration) (*Cache, error) {
if err := schema.Migrate(d.Primary(), schema.Postgres, schemaStore, postgresCurrentSchemaVersion, postgresCreateTables, postgresMigrations); err != nil {
if err := schema.Migrate(d.Primary(), schema.Postgres, schemaStore, postgresCreateTables, postgresMigrations); err != nil {
return nil, err
}
return newCache(d, postgresQueries, nil, batchSize, batchTimeout, false), nil
}
// postgresInsertMessages transposes the batch into one array per column and writes it with
// postgresInsertMessagesQuery in a single round trip
func postgresInsertMessages(tx *sql.Tx, ms []*model.Message) error {
columns := make([][]any, insertMessageColumns)
for _, m := range ms {
args, err := insertMessageArgs(m)
if err != nil {
return err
}
for i, arg := range args {
columns[i] = append(columns[i], arg)
}
}
params := make([]any, 0, len(columns))
for _, column := range columns {
params = append(params, column)
}
_, err := tx.Exec(postgresInsertMessagesQuery, params...)
return err
}
message/cache_postgres_internal_test.go
+121
View File
@@ -0,0 +1,121 @@
package message
import (
"context"
"database/sql"
"regexp"
"strings"
"testing"
"github.com/jackc/pgx/v5/stdlib"
"github.com/stretchr/testify/require"
dbtest "heckel.io/ntfy/v2/db/test"
)
// The since=<id> replay and the tail read must start every scan of the message table at the marker:
// the index condition needs a lower bound (id >, time >=) or an exact set of ids (= ANY). Narrowed by
// topic alone, or bounded only from above (id <), a scan reads the topic's whole history: on 2026-10-16
// (v2.31.0) an OR in the replay query made every reconnect read its whole topic, and the reconnects of
// ntfy.sh's subscribers saturated a read replica.
//
// The plans checked are the generic ones: the driver prepares its statements, and after five
// executions Postgres may switch to a plan made without the parameter values. That plan is the one
// that read the whole topic; with the values, the planner had found a cheaper plan.
func TestPostgresQueries_GenericPlansUseOnlyIndexConditions(t *testing.T) {
d := dbtest.CreateTestPostgres(t)
_, err := NewPostgresStore(d, 0, 0)
require.Nil(t, err)
// Rows and statistics shaped like ntfy.sh's cache: a few busy topics (6% of the rows each) among many
// small ones, written in time order. The planner picks plans by these statistics: on a small or even
// table, it finds cheap plans even for the queries that read whole topics on a real server.
_, err = d.Primary().Exec(`
INSERT INTO message (mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions,
attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user_id, content_type, encoding, published)
SELECT 'm' || g, 'm' || g, 1792100000 + g / 5, 'message', 1792200000 + g, CASE
WHEN g % 100 < 6 THEN 'mytopic'
WHEN g % 100 < 30 THEN 'busy' || (g % 4)
ELSE 'small' || (g % 20000)
END,
'body', '', 3, '', '', '', '', '', '', 0, 0, '', '', '', '', '', TRUE
FROM generate_series(1, 200000) g;
ANALYZE message`)
require.Nil(t, err)
for name, q := range map[string]struct {
query string
args string
topicScoped bool // A replay reads one topic, the tail all of them
}{
"since id": {postgresQueries.selectMessagesSinceID, `'mytopic', 1000, 1792160000`, true},
"since id, include scheduled": {postgresQueries.selectMessagesSinceIDScheduled, `'mytopic', 1000, 1792160000`, true},
"tail": {postgresQueries.selectMessagesAfterRowID, `1000, '[998, 999]', 1000`, false},
} {
plan := postgresGenericPlan(t, d.Primary(), q.query, q.args)
require.NotContains(t, plan, "Seq Scan on message", "%s: reads the whole table:\n%s", name, plan)
scans := postgresIndexScanRe.FindAllStringSubmatch(plan, -1)
require.NotEmpty(t, scans, "%s: no index scan:\n%s", name, plan)
for _, scan := range scans {
index, condition := scan[1], scan[2]
bounds, known := postgresScanBounds[index]
require.True(t, known, "%s: scan on unexpected index %s:\n%s", name, index, plan)
require.False(t, q.topicScoped && index == "message_pkey", "%s: a scan reads other topics' rows:\n%s", name, plan)
require.Regexp(t, bounds, condition, "%s: the scan on %s does not start at the marker:\n%s", name, index, plan)
}
}
}
func TestPostgresStore_CreatesTopicPublishedIDIndex(t *testing.T) {
// since=<id> replays read the rows after the marker through (topic, published, id); without it,
// v2.30.0's replay walked every index entry of the topic (ntfy.sh, 2026-10-16)
d := dbtest.CreateTestPostgres(t)
_, err := NewPostgresStore(d, 0, 0)
require.Nil(t, err)
var definition string
// pg_indexes spans every schema: without the schema filter, a parallel test dropping its schema fails this one
require.Nil(t, d.Primary().QueryRow(`SELECT indexdef FROM pg_indexes WHERE schemaname = current_schema() AND tablename = 'message' AND indexname = 'idx_message_topic_published_id'`).Scan(&definition))
require.Contains(t, definition, "(topic, published, id)")
}
var (
// postgresIndexScanRe matches an index scan and its index condition (these queries read only the message table)
postgresIndexScanRe = regexp.MustCompile(`Index (?:Only )?Scan(?: Backward)? (?:using|on) (\S+)[^\n]*\n\s+Index Cond: ([^\n]+)`)
// postgresScanBounds are the conditions that start a scan of each index at the marker. A condition on a
// later index column does not: on (topic, published, time, id), "id > $2" still walks the whole topic.
// Unpublished (scheduled) messages are listed in full, and they are few on a whole server.
postgresScanBounds = map[string]*regexp.Regexp{
"idx_message_topic_published_time": regexp.MustCompile(`"time" >= |published = false`),
"idx_message_topic_published_id": regexp.MustCompile(`id > |published = false`),
"idx_message_published_expires": regexp.MustCompile(`published = false`),
"message_pkey": regexp.MustCompile(`id > |id = ANY`),
}
)
// postgresGenericPlan returns the generic plan of a query, prepared and explained over the simple
// protocol, which passes $1, $2, ... through to Postgres instead of binding them
func postgresGenericPlan(t *testing.T, db *sql.DB, query, args string) string {
conn, err := db.Conn(context.Background())
require.Nil(t, err)
defer conn.Close()
var plan string
require.Nil(t, conn.Raw(func(driverConn any) error {
pgConn := driverConn.(*stdlib.Conn).Conn().PgConn()
results, err := pgConn.Exec(context.Background(), `
DEALLOCATE ALL; -- The pool reuses connections
SET enable_seqscan = off; -- A small test table would otherwise be read sequentially
SET plan_cache_mode = force_generic_plan;
PREPARE q AS `+query+`;
EXPLAIN EXECUTE q(`+args+`);
`).ReadAll()
if err != nil {
return err
}
lines := make([]string, 0)
for _, row := range results[len(results)-1].Rows {
lines = append(lines, string(row[0]))
}
plan = strings.Join(lines, "\n")
return nil
}))
return plan
}
message/cache_postgres_schema.go
+18 -5
View File
@@ -6,8 +6,7 @@ import (
// Initial PostgreSQL schema
const (
postgresCurrentSchemaVersion = 15
postgresCreateTablesQuery = `
postgresCreateTablesQuery = `
CREATE TABLE IF NOT EXISTS message (
id BIGSERIAL PRIMARY KEY,
mid TEXT NOT NULL,
@@ -33,11 +32,13 @@ const (
user_id TEXT NOT NULL,
content_type TEXT NOT NULL,
encoding TEXT NOT NULL,
published BOOLEAN NOT NULL DEFAULT FALSE
published BOOLEAN NOT NULL DEFAULT FALSE,
claimed_at BIGINT NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_message_mid ON message (mid);
CREATE INDEX IF NOT EXISTS idx_message_sequence_id ON message (sequence_id);
CREATE INDEX IF NOT EXISTS idx_message_topic_published_time ON message (topic, published, time, id);
CREATE INDEX IF NOT EXISTS idx_message_topic_published_id ON message (topic, published, id);
CREATE INDEX IF NOT EXISTS idx_message_published_expires ON message (published, expires);
CREATE INDEX IF NOT EXISTS idx_message_attachment_expires ON message (attachment_expires) WHERE attachment_deleted = FALSE;
CREATE INDEX IF NOT EXISTS idx_message_sender_attachment_expires ON message (sender, attachment_expires) WHERE user_id = '';
@@ -56,6 +57,16 @@ const (
postgresMigrate14To15CreateIndexQuery = `
CREATE INDEX IF NOT EXISTS idx_message_attachment_expires ON message (attachment_expires) WHERE attachment_deleted = FALSE;
`
// 15 -> 16
postgresMigrate15To16AddClaimedAtQuery = `
ALTER TABLE message ADD COLUMN IF NOT EXISTS claimed_at BIGINT NOT NULL DEFAULT 0;
`
// 16 -> 17: serves the rows after a since=<id> marker; ntfy.sh got it by hand on 2026-10-16, which IF NOT EXISTS keeps
postgresMigrate16To17CreateIndexQuery = `
CREATE INDEX IF NOT EXISTS idx_message_topic_published_id ON message (topic, published, id);
`
)
var (
@@ -63,7 +74,9 @@ var (
// postgresMigrations maps a schema version to the migration upgrading it to the next
// version. Always append migrations at the end, never insert in the middle.
postgresMigrations = map[int]schema.MigrateFunc{
14: schema.AsMigrateFunc(postgresMigrate14To15CreateIndexQuery),
postgresMigrations = schema.Migrations{
{From: 14, To: 15}: schema.Query(postgresMigrate14To15CreateIndexQuery),
{From: 15, To: 16}: schema.Query(postgresMigrate15To16AddClaimedAtQuery),
{From: 16, To: 17}: schema.Query(postgresMigrate16To17CreateIndexQuery),
}
)
message/cache_postgres_test.go
+6 -2
View File
@@ -55,10 +55,14 @@ func TestPostgresStore_Migration_From14(t *testing.T) {
require.Nil(t, err)
store, err := message.NewPostgresStore(testDB, 0, 0)
require.Nil(t, err)
// The 14 -> 15 step ran: version bumped, partial index created
// The 14 -> 15, 15 -> 16 and 16 -> 17 steps ran: version bumped, partial index created, claim column
// added, (topic, published, id) index created
var version int
require.Nil(t, testDB.QueryRow(`SELECT version FROM schema_version WHERE store = 'message'`).Scan(&version))
require.Equal(t, 15, version)
require.Equal(t, 17, version)
var claimedAtCount int
require.Nil(t, testDB.QueryRow(`SELECT COUNT(*) FROM information_schema.columns WHERE table_name = 'message' AND column_name = 'claimed_at' AND table_schema = current_schema()`).Scan(&claimedAtCount))
require.Equal(t, 1, claimedAtCount)
var indexCount int
require.Nil(t, testDB.QueryRow(`SELECT COUNT(*) FROM pg_indexes WHERE indexname = 'idx_message_attachment_expires' AND schemaname = current_schema()`).Scan(&indexCount))
require.Equal(t, 1, indexCount)
message/cache_sqlite.go
+63 -11
View File
@@ -4,21 +4,26 @@ import (
"database/sql"
"fmt"
"path/filepath"
"strings"
"sync"
"time"
_ "github.com/mattn/go-sqlite3" // SQLite driver
"heckel.io/ntfy/v2/db"
"heckel.io/ntfy/v2/db/schema"
"heckel.io/ntfy/v2/model"
"heckel.io/ntfy/v2/util"
)
// SQLite runtime query constants
const (
sqliteInsertMessageQuery = `
INSERT INTO messages (mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_deleted, sender, user, content_type, encoding, published)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`
// sqliteInsertMessagesQuery is the head of the multi-row INSERT; one sqliteInsertMessagesRow per
// message follows, with the values in the order of insertMessageArgs. sqliteInsertMessagesMaxRows
// caps the rows per statement, keeping the parameter count well below SQLite's 32766.
sqliteInsertMessagesQuery = `INSERT INTO messages (mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_deleted, sender, user, content_type, encoding, published) VALUES `
sqliteInsertMessagesRow = `(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
sqliteInsertMessagesMaxRows = 1000
sqliteSelectScheduledMessageIDsBySeqIDQuery = `SELECT mid FROM messages WHERE topic = ? AND sequence_id = ? AND published = 0`
sqliteDeleteScheduledBySequenceIDQuery = `DELETE FROM messages WHERE topic = ? AND sequence_id = ? AND published = 0`
sqliteUpdateMessagesForTopicExpiryQuery = `UPDATE messages SET expires = ? WHERE topic = ?`
@@ -39,16 +44,38 @@ const (
WHERE topic = ? AND time >= ?
ORDER BY time DESC, id DESC
`
sqliteSelectMessageRowIDQuery = `SELECT id, time FROM messages WHERE mid = ? LIMIT 1`
sqliteSelectRecentRowIDsQuery = `SELECT id FROM messages ORDER BY id DESC LIMIT ?`
// sqliteSelectMessagesAfterRowIDQuery: see postgresSelectMessagesAfterRowIDQuery
sqliteSelectMessagesAfterRowIDQuery = `
SELECT mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding, id, published
FROM (
SELECT * FROM messages WHERE id > ?1
UNION ALL
SELECT * FROM messages WHERE id IN (SELECT value FROM json_each(?2)) AND id <= ?1
)
ORDER BY id
LIMIT ?3
`
// sqliteSelectMessagesSinceIDQuery: see postgresSelectMessagesSinceIDQuery
sqliteSelectMessagesSinceIDQuery = `
SELECT mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding
FROM messages
WHERE topic = ? AND id > COALESCE((SELECT id FROM messages WHERE mid = ?), 0) AND published = 1
FROM (
SELECT * FROM messages WHERE topic = ?1 AND published = 1 AND time >= ?3 AND id <> ?2
UNION ALL
SELECT * FROM messages WHERE topic = ?1 AND published = 1 AND id > ?2 AND time + 0 < ?3
)
ORDER BY time DESC, id DESC
`
sqliteSelectMessagesSinceIDIncludeScheduledQuery = `
SELECT mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding
FROM messages
WHERE topic = ? AND (id > COALESCE((SELECT id FROM messages WHERE mid = ?), 0) OR published = 0)
FROM (
SELECT * FROM messages WHERE topic = ?1 AND published = 1 AND time >= ?3 AND id <> ?2
UNION ALL
SELECT * FROM messages WHERE topic = ?1 AND published = 1 AND id > ?2 AND time + 0 < ?3
UNION ALL
SELECT * FROM messages WHERE topic = ?1 AND published = 0
)
ORDER BY time DESC, id DESC
`
sqliteSelectMessagesLatestQuery = `
@@ -75,20 +102,23 @@ const (
sqliteSelectAttachmentsWithSizesQuery = `SELECT mid, attachment_size FROM messages WHERE attachment_expires > ? AND attachment_deleted = 0`
sqliteSelectStatsQuery = `SELECT value FROM stats WHERE key = 'messages'`
sqliteUpdateStatsQuery = `UPDATE stats SET value = ? WHERE key = 'messages'`
sqliteUpdateStatsQuery = `UPDATE stats SET value = value + ? WHERE key = 'messages'`
sqliteUpdateMessageTimeQuery = `UPDATE messages SET time = ? WHERE mid = ?`
)
var sqliteQueries = queries{
insertMessage: sqliteInsertMessageQuery,
insertMessages: sqliteInsertMessages,
selectScheduledMessageIDsBySeqID: sqliteSelectScheduledMessageIDsBySeqIDQuery,
deleteScheduledBySequenceID: sqliteDeleteScheduledBySequenceIDQuery,
updateMessagesForTopicExpiry: sqliteUpdateMessagesForTopicExpiryQuery,
selectMessagesByID: sqliteSelectMessagesByIDQuery,
selectMessagesSinceTime: sqliteSelectMessagesSinceTimeQuery,
selectMessagesSinceTimeScheduled: sqliteSelectMessagesSinceTimeIncludeScheduledQuery,
selectMessageRowID: sqliteSelectMessageRowIDQuery,
selectMessagesSinceID: sqliteSelectMessagesSinceIDQuery,
selectMessagesSinceIDScheduled: sqliteSelectMessagesSinceIDIncludeScheduledQuery,
selectRecentRowIDs: sqliteSelectRecentRowIDsQuery,
selectMessagesAfterRowID: sqliteSelectMessagesAfterRowIDQuery,
selectMessagesLatest: sqliteSelectMessagesLatestQuery,
selectMessagesDue: sqliteSelectMessagesDueQuery,
deleteExpiredMessages: sqliteDeleteExpiredMessagesQuery,
@@ -117,7 +147,7 @@ func NewSQLiteStore(filename, startupQueries string, cacheDuration time.Duration
if err := runSQLiteStartupQueries(d, startupQueries); err != nil {
return nil, err
}
if err := schema.Migrate(d, schema.SQLite, schemaStore, sqliteCurrentSchemaVersion, sqliteCreateTables, sqliteMigrations(cacheDuration)); err != nil {
if err := schema.Migrate(d, schema.SQLite, schemaStore, sqliteCreateTables, sqliteMigrations(cacheDuration)); err != nil {
return nil, err
}
return newCache(db.New(&db.Host{DB: d}, nil), sqliteQueries, &sync.Mutex{}, batchSize, batchTimeout, nop), nil
@@ -143,3 +173,25 @@ func NewNopStore() (*Cache, error) {
func createMemoryFilename() string {
return fmt.Sprintf("file:%s?mode=memory&cache=shared", util.RandomString(10))
}
// sqliteInsertMessages writes the batch as multi-row INSERTs of at most sqliteInsertMessagesMaxRows
// messages each
func sqliteInsertMessages(tx *sql.Tx, ms []*model.Message) error {
for len(ms) > 0 {
chunk := ms[:min(len(ms), sqliteInsertMessagesMaxRows)]
ms = ms[len(chunk):]
args := make([]any, 0, len(chunk)*insertMessageColumns)
for _, m := range chunk {
rowArgs, err := insertMessageArgs(m)
if err != nil {
return err
}
args = append(args, rowArgs...)
}
query := sqliteInsertMessagesQuery + strings.Repeat(sqliteInsertMessagesRow+", ", len(chunk)-1) + sqliteInsertMessagesRow
if _, err := tx.Exec(query, args...); err != nil {
return err
}
}
return nil
}
message/cache_sqlite_internal_test.go
+42
View File
@@ -0,0 +1,42 @@
package message
import (
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
// See TestPostgresQueries_GenericPlansUseOnlyIndexConditions
func TestSQLiteQueries_SinceIDAndTailUseOnlyIndexConditions(t *testing.T) {
// SQLite plans without the parameter values, so this is the plan it always uses
c, err := NewSQLiteStore(filepath.Join(t.TempDir(), "cache.db"), "", 0, 0, 0, false)
require.Nil(t, err)
defer c.Close()
for name, q := range map[string]struct {
query string
args []any
}{
"since id": {sqliteQueries.selectMessagesSinceID, []any{"mytopic", 1000, 1792160000}},
"since id, include scheduled": {sqliteQueries.selectMessagesSinceIDScheduled, []any{"mytopic", 1000, 1792160000}},
"tail": {sqliteQueries.selectMessagesAfterRowID, []any{1000, "[998, 999]", 1000}},
} {
rows, err := c.db.Primary().Query("EXPLAIN QUERY PLAN "+q.query, q.args...)
require.Nil(t, err)
plan := make([]string, 0)
for rows.Next() {
var id, parent, notUsed int
var detail string
require.Nil(t, rows.Scan(&id, &parent, &notUsed, &detail))
plan = append(plan, detail)
}
require.Nil(t, rows.Close())
for _, step := range plan {
// "(topic=?)" alone reads every row of the topic, and "rowid<?" every row written before the marker
require.False(t, strings.HasPrefix(step, "SCAN messages"), "%s: full table scan:\n%s", name, strings.Join(plan, "\n"))
require.False(t, strings.HasSuffix(step, "(topic=?)"), "%s: reads the whole topic:\n%s", name, strings.Join(plan, "\n"))
require.False(t, strings.HasSuffix(step, "(topic=? AND rowid<?)"), "%s: reads the topic's history:\n%s", name, strings.Join(plan, "\n"))
}
}
}
message/cache_sqlite_schema.go
+26 -19
View File
@@ -9,8 +9,7 @@ import (
// Initial SQLite schema
const (
sqliteCurrentSchemaVersion = 15
sqliteCreateTablesQuery = `
sqliteCreateTablesQuery = `
CREATE TABLE IF NOT EXISTS messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
mid TEXT NOT NULL,
@@ -42,6 +41,7 @@ const (
CREATE INDEX IF NOT EXISTS idx_sequence_id ON messages (sequence_id);
CREATE INDEX IF NOT EXISTS idx_time ON messages (time);
CREATE INDEX IF NOT EXISTS idx_topic ON messages (topic);
CREATE INDEX IF NOT EXISTS idx_topic_published_time ON messages (topic, published, time);
CREATE INDEX IF NOT EXISTS idx_expires ON messages (expires);
CREATE INDEX IF NOT EXISTS idx_sender ON messages (sender);
CREATE INDEX IF NOT EXISTS idx_user ON messages (user);
@@ -169,6 +169,11 @@ const (
ALTER TABLE messages ADD COLUMN event TEXT NOT NULL DEFAULT('message');
CREATE INDEX IF NOT EXISTS idx_sequence_id ON messages (sequence_id);
`
// 16 -> 17: serves the since=<id> replay, like Postgres' idx_message_topic_published_time
sqliteMigrate16To17CreateIndexQuery = `
CREATE INDEX IF NOT EXISTS idx_topic_published_time ON messages (topic, published, time);
`
)
var (
@@ -178,28 +183,30 @@ var (
// sqliteMigrations returns the migration steps, keyed by the version they upgrade FROM. The
// cache duration is carried into the 9 -> 10 step via closure (it backfills "expires" from it).
// Always append migrations at the end, never insert in the middle.
func sqliteMigrations(cacheDuration time.Duration) map[int]schema.MigrateFunc {
return map[int]schema.MigrateFunc{
1: schema.AsMigrateFunc(sqliteMigrate1To2AlterMessagesTableQuery),
2: schema.AsMigrateFunc(sqliteMigrate2To3AlterMessagesTableQuery),
3: schema.AsMigrateFunc(sqliteMigrate3To4AlterMessagesTableQuery),
4: schema.AsMigrateFunc(sqliteMigrate4To5AlterMessagesTableQuery),
5: schema.AsMigrateFunc(sqliteMigrate5To6AlterMessagesTableQuery),
6: schema.AsMigrateFunc(sqliteMigrate6To7AlterMessagesTableQuery),
7: schema.AsMigrateFunc(sqliteMigrate7To8AlterMessagesTableQuery),
8: schema.AsMigrateFunc(sqliteMigrate8To9AlterMessagesTableQuery),
9: func(tx *sql.Tx) error {
func sqliteMigrations(cacheDuration time.Duration) schema.Migrations {
return schema.Migrations{
{From: 1, To: 2}: schema.Query(sqliteMigrate1To2AlterMessagesTableQuery),
{From: 2, To: 3}: schema.Query(sqliteMigrate2To3AlterMessagesTableQuery),
{From: 3, To: 4}: schema.Query(sqliteMigrate3To4AlterMessagesTableQuery),
{From: 4, To: 5}: schema.Query(sqliteMigrate4To5AlterMessagesTableQuery),
{From: 5, To: 6}: schema.Query(sqliteMigrate5To6AlterMessagesTableQuery),
{From: 6, To: 7}: schema.Query(sqliteMigrate6To7AlterMessagesTableQuery),
{From: 7, To: 8}: schema.Query(sqliteMigrate7To8AlterMessagesTableQuery),
{From: 8, To: 9}: schema.Query(sqliteMigrate8To9AlterMessagesTableQuery),
{From: 9, To: 10}: schema.Func(func(tx *sql.Tx) error {
if _, err := tx.Exec(sqliteMigrate9To10AlterMessagesTableQuery); err != nil {
return err
}
_, err := tx.Exec(sqliteMigrate9To10UpdateMessageExpiryQuery, int64(cacheDuration.Seconds()))
return err
},
10: schema.AsMigrateFunc(sqliteMigrate10To11AlterMessagesTableQuery),
11: schema.AsMigrateFunc(sqliteMigrate11To12AlterMessagesTableQuery),
12: schema.AsMigrateFunc(sqliteMigrate12To13AlterMessagesTableQuery),
13: schema.AsMigrateFunc(sqliteMigrate13To14AlterMessagesTableQuery),
14: schema.NopMigrateFunc, // Corresponds to Postgres migration
}),
{From: 10, To: 11}: schema.Query(sqliteMigrate10To11AlterMessagesTableQuery),
{From: 11, To: 12}: schema.Query(sqliteMigrate11To12AlterMessagesTableQuery),
{From: 12, To: 13}: schema.Query(sqliteMigrate12To13AlterMessagesTableQuery),
{From: 13, To: 14}: schema.Query(sqliteMigrate13To14AlterMessagesTableQuery),
{From: 14, To: 15}: schema.Nop, // Corresponds to Postgres migration
{From: 15, To: 16}: schema.Nop, // 15 -> 16 adds the Postgres claim column; SQLite is single-node
{From: 16, To: 17}: schema.Query(sqliteMigrate16To17CreateIndexQuery),
}
}
message/cache_sqlite_test.go
+19 -3
View File
@@ -183,7 +183,7 @@ func TestSqliteStore_Migration_From9(t *testing.T) {
require.True(t, rows.Next())
var version int
require.Nil(t, rows.Scan(&version))
require.Equal(t, 15, version)
require.Equal(t, 17, version)
require.Nil(t, rows.Close())
messages, err := s.Messages("mytopic", model.SinceAllMessages, false)
@@ -256,11 +256,27 @@ func checkSqliteSchemaVersion(t *testing.T, filename string) {
db, err := sql.Open("sqlite3", filename)
require.Nil(t, err)
defer db.Close()
rows, err := db.Query(`SELECT version FROM schemaVersion`)
rows, err := db.Query(`SELECT version FROM schemaVersion WHERE id = 1`)
require.Nil(t, err)
require.True(t, rows.Next())
var schemaVersion int
require.Nil(t, rows.Scan(&schemaVersion))
require.Equal(t, 15, schemaVersion)
require.Equal(t, 17, schemaVersion)
require.Nil(t, rows.Close())
}
func TestSqliteStore_CloseFlushesQueuedMessages(t *testing.T) {
// Messages accepted into the write batch must reach the database on a graceful Close;
// before, a restart dropped up to one batch timeout's worth of accepted messages
filename := newSqliteTestStoreFile(t)
s, err := message.NewSQLiteStore(filename, "", time.Hour, 100, time.Hour, false)
require.Nil(t, err)
require.Nil(t, s.AddMessage(model.NewDefaultMessage("mytopic", "queued 1")))
require.Nil(t, s.AddMessage(model.NewDefaultMessage("mytopic", "queued 2")))
require.Nil(t, s.Close())
reopened := newSqliteTestStoreFromFile(t, filename, "")
messages, err := reopened.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, 2, len(messages))
}
message/cache_test.go
+414 -19
View File
@@ -1,6 +1,7 @@
package message_test
import (
"fmt"
"net/netip"
"path/filepath"
"sync"
@@ -9,6 +10,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/db"
"heckel.io/ntfy/v2/db/pg"
dbtest "heckel.io/ntfy/v2/db/test"
"heckel.io/ntfy/v2/message"
"heckel.io/ntfy/v2/model"
@@ -85,7 +88,7 @@ func TestStore_Messages(t *testing.T) {
require.Empty(t, messages)
// mytopic: since m1 (by ID)
messages, _ = s.Messages("mytopic", model.NewSinceID(m1.ID), false)
messages = messagesAfter(t, s, "mytopic", m1.ID, false)
require.Equal(t, 1, len(messages))
require.Equal(t, m2.ID, messages[0].ID)
require.Equal(t, "my other message", messages[0].Message)
@@ -211,14 +214,14 @@ func TestStore_MessagesSinceID(t *testing.T) {
require.Nil(t, s.AddMessage(m7))
// Case 1: Since ID exists, exclude scheduled
messages, _ := s.Messages("mytopic", model.NewSinceID(m2.ID), false)
messages := messagesAfter(t, s, "mytopic", m2.ID, false)
require.Equal(t, 3, len(messages))
require.Equal(t, "message 4", messages[0].Message)
require.Equal(t, "message 6", messages[1].Message) // Not scheduled m3/m5!
require.Equal(t, "message 7", messages[2].Message)
// Case 2: Since ID exists, include scheduled
messages, _ = s.Messages("mytopic", model.NewSinceID(m2.ID), true)
messages = messagesAfter(t, s, "mytopic", m2.ID, true)
require.Equal(t, 5, len(messages))
require.Equal(t, "message 4", messages[0].Message)
require.Equal(t, "message 6", messages[1].Message)
@@ -226,23 +229,17 @@ func TestStore_MessagesSinceID(t *testing.T) {
require.Equal(t, "message 5", messages[3].Message) // Order!
require.Equal(t, "message 3", messages[4].Message) // Order!
// Case 3: Since ID does not exist (-> Return all messages), include scheduled
messages, _ = s.Messages("mytopic", model.NewSinceID("doesntexist"), true)
require.Equal(t, 7, len(messages))
require.Equal(t, "message 1", messages[0].Message)
require.Equal(t, "message 2", messages[1].Message)
require.Equal(t, "message 4", messages[2].Message)
require.Equal(t, "message 6", messages[3].Message)
require.Equal(t, "message 7", messages[4].Message)
require.Equal(t, "message 5", messages[5].Message) // Order!
require.Equal(t, "message 3", messages[6].Message) // Order!
// Case 3: Since ID does not exist: the caller decides (the server replays everything)
marker, err := s.ResolveMarker("doesntexist")
require.Nil(t, err)
require.Nil(t, marker)
// Case 4: Since ID exists and is last message (-> Return no messages), exclude scheduled
messages, _ = s.Messages("mytopic", model.NewSinceID(m7.ID), false)
messages = messagesAfter(t, s, "mytopic", m7.ID, false)
require.Equal(t, 0, len(messages))
// Case 5: Since ID exists and is last message (-> Return no messages), include scheduled
messages, _ = s.Messages("mytopic", model.NewSinceID(m7.ID), true)
messages = messagesAfter(t, s, "mytopic", m7.ID, true)
require.Equal(t, 2, len(messages))
require.Equal(t, "message 5", messages[0].Message)
require.Equal(t, "message 3", messages[1].Message)
@@ -556,6 +553,76 @@ func TestStore_MarkPublished(t *testing.T) {
})
}
func TestStore_MessagesDue_ClaimExactlyOnce(t *testing.T) {
// Postgres-only: exercises "FOR UPDATE SKIP LOCKED" claiming so that concurrent delayed
// senders running on different cluster nodes never pick up (and deliver) the same due
// message twice. With the non-claiming implementation, every concurrent caller sees every
// due row, so this test fails; with claiming, each row is returned to exactly one caller.
s := newTestPostgresStore(t) // skips if NTFY_TEST_DATABASE_URL is unset
const n = 40
for i := 0; i < n; i++ {
m := model.NewDefaultMessage("mytopic", fmt.Sprintf("scheduled %d", i))
m.Time = time.Now().Add(time.Hour).Unix() // future -> stored as published=FALSE
require.Nil(t, s.AddMessage(m))
// Move the time into the past so the message is due now (but still unpublished)
require.Nil(t, s.UpdateMessageTime(m.ID, time.Now().Add(-time.Minute).Unix()))
}
var mu sync.Mutex
seen := make(map[string]int)
var wg sync.WaitGroup
for c := 0; c < 6; c++ {
wg.Add(1)
go func() {
defer wg.Done()
due, err := s.MessagesDue()
require.Nil(t, err)
mu.Lock()
defer mu.Unlock()
for _, m := range due {
seen[m.ID]++
}
}()
}
wg.Wait()
require.Len(t, seen, n) // every due message was claimed
for id, count := range seen {
require.Equalf(t, 1, count, "message %s was claimed %d times, want exactly 1", id, count)
}
}
func TestStore_MessagesDue_AbandonedClaimIsReclaimed(t *testing.T) {
// A node that dies between claiming and delivering leaves its rows claimed but unpublished.
// Nothing would ever deliver them again if the claim were permanent, so it expires: until it
// does, no other node sees the message; after it does, any node may claim it afresh.
s := newTestPostgresStore(t) // skips if NTFY_TEST_DATABASE_URL is unset
m := model.NewDefaultMessage("mytopic", "scheduled")
m.Time = time.Now().Add(time.Hour).Unix() // future -> stored unpublished
require.Nil(t, s.AddMessage(m))
require.Nil(t, s.UpdateMessageTime(m.ID, time.Now().Add(-time.Minute).Unix()))
// The claim takes it, and nothing else can have it while the claim holds
due, err := s.MessagesDue()
require.Nil(t, err)
require.Len(t, due, 1)
require.Equal(t, m.ID, due[0].ID)
due, err = s.MessagesDue()
require.Nil(t, err)
require.Empty(t, due, "a claimed message was handed out again while the claim still held")
// The claimer never published it (it died); past the timeout the message is claimable again
s.SetClaimTimeoutForTest(0)
due, err = s.MessagesDue()
require.Nil(t, err)
require.Len(t, due, 1, "an abandoned claim was never reclaimed")
require.Equal(t, m.ID, due[0].ID)
// Once it is actually delivered, no timeout brings it back
require.Nil(t, s.MarkPublished(due[0]))
due, err = s.MessagesDue()
require.Nil(t, err)
require.Empty(t, due, "a published message was handed out again")
}
func TestStore_ExpireMessages(t *testing.T) {
forEachBackend(t, func(t *testing.T, s *message.Cache) {
// Add messages to two topics
@@ -647,14 +714,14 @@ func TestStore_Stats(t *testing.T) {
require.Nil(t, err)
require.Equal(t, int64(0), messages)
// Update stats
require.Nil(t, s.UpdateStats(42))
// Add stats
require.Nil(t, s.AddStats(42))
messages, err = s.Stats()
require.Nil(t, err)
require.Equal(t, int64(42), messages)
// Update again (overwrites)
require.Nil(t, s.UpdateStats(100))
// Add again (increments)
require.Nil(t, s.AddStats(58))
messages, err = s.Stats()
require.Nil(t, err)
require.Equal(t, int64(100), messages)
@@ -692,6 +759,101 @@ func TestStore_AddMessages(t *testing.T) {
})
}
func TestStore_AddMessages_AllFields(t *testing.T) {
forEachBackend(t, func(t *testing.T, s *message.Cache) {
// Every column must land in the right row when several messages are written in one batch
msgs := make([]*model.Message, 0)
for i := 0; i < 3; i++ {
m := model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i))
m.Time = int64(1000 + i)
m.SequenceID = fmt.Sprintf("seq%d", i)
m.Title = fmt.Sprintf("title %d", i)
m.Priority = i + 1
m.Tags = []string{fmt.Sprintf("tag%d", i), "common"}
m.Click = fmt.Sprintf("https://example.com/%d", i)
m.Icon = fmt.Sprintf("https://example.com/%d.png", i)
m.Actions = []*model.Action{{ID: fmt.Sprintf("a%d", i), Action: "view", Label: fmt.Sprintf("label %d", i), URL: "https://example.com"}}
m.Attachment = &model.Attachment{Name: fmt.Sprintf("file%d.txt", i), Type: "text/plain", Size: int64(10 + i), Expires: int64(2000 + i), URL: fmt.Sprintf("https://example.com/file%d.txt", i)}
m.Sender = netip.MustParseAddr(fmt.Sprintf("1.2.3.%d", i+1))
m.User = fmt.Sprintf("u_%d", i)
m.ContentType = "text/markdown"
m.Encoding = "base64"
msgs = append(msgs, m)
}
require.Nil(t, s.AddMessages(msgs))
messages, err := s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, 3, len(messages))
for i, m := range messages {
require.Equal(t, msgs[i].ID, m.ID)
require.Equal(t, fmt.Sprintf("seq%d", i), m.SequenceID)
require.Equal(t, int64(1000+i), m.Time)
require.Equal(t, fmt.Sprintf("message %d", i), m.Message)
require.Equal(t, fmt.Sprintf("title %d", i), m.Title)
require.Equal(t, i+1, m.Priority)
require.Equal(t, []string{fmt.Sprintf("tag%d", i), "common"}, m.Tags)
require.Equal(t, fmt.Sprintf("https://example.com/%d", i), m.Click)
require.Equal(t, fmt.Sprintf("https://example.com/%d.png", i), m.Icon)
require.Equal(t, 1, len(m.Actions))
require.Equal(t, fmt.Sprintf("label %d", i), m.Actions[0].Label)
require.Equal(t, fmt.Sprintf("file%d.txt", i), m.Attachment.Name)
require.Equal(t, int64(10+i), m.Attachment.Size)
require.Equal(t, int64(2000+i), m.Attachment.Expires)
require.Equal(t, fmt.Sprintf("https://example.com/file%d.txt", i), m.Attachment.URL)
require.Equal(t, fmt.Sprintf("1.2.3.%d", i+1), m.Sender.String())
require.Equal(t, fmt.Sprintf("u_%d", i), m.User)
require.Equal(t, "text/markdown", m.ContentType)
require.Equal(t, "base64", m.Encoding)
}
})
}
func TestPostgresStore_AddMessages_OneRoundTripPerBatch(t *testing.T) {
// The batch writer is the only writer, so its round trips cap the publish rate on cron bursts.
// With one INSERT per row, a batch of 100 against a remote database costs 100+ round trips.
testDB, proxy := dbtest.CreateTestPostgresWithLatency(t, time.Millisecond)
s, err := message.NewPostgresStore(testDB, 0, 0)
require.Nil(t, err)
msgs := make([]*model.Message, 0)
for i := 0; i < 100; i++ {
msgs = append(msgs, model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i)))
}
require.Nil(t, s.AddMessages(msgs[:1])) // Warm up the pool connection
before := proxy.Requests()
require.Nil(t, s.AddMessages(msgs[1:]))
roundTrips := proxy.Requests() - before
require.Less(t, roundTrips, int64(10), "writing 99 messages took %d round trips", roundTrips)
messages, err := s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, 100, len(messages))
}
func TestPostgresStore_AddMessages_OnePreparedStatementForAnyBatchSize(t *testing.T) {
// pgx prepares every distinct statement text server-side and keeps up to 512 per connection.
// A multi-row INSERT whose text depends on the batch size leaves hundreds of multi-MB plans
// in every pooled backend (ntfy.sh primary memory crept from 50% to 65% in 3h after 2.29.0).
testDB := dbtest.CreateTestPostgres(t)
testDB.Primary().SetMaxOpenConns(1) // Every statement lands on the same backend
s, err := message.NewPostgresStore(testDB, 0, 0)
require.Nil(t, err)
for _, size := range []int{1, 2, 50, 300} {
msgs := make([]*model.Message, 0)
for i := 0; i < size; i++ {
msgs = append(msgs, model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i)))
}
require.Nil(t, s.AddMessages(msgs))
}
var prepared int
require.Nil(t, testDB.Primary().QueryRow("SELECT COUNT(*) FROM pg_prepared_statements WHERE statement ~ '^\\s*INSERT INTO message '").Scan(&prepared)) // Anchored, so this SELECT does not count itself
require.Equal(t, 1, prepared, "one INSERT shape regardless of batch size")
messages, err := s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, 353, len(messages))
}
func TestStore_MessagesDue(t *testing.T) {
forEachBackend(t, func(t *testing.T, s *message.Cache) {
// Add a message scheduled in the past (i.e. it's due now)
@@ -947,3 +1109,236 @@ func TestStore_AddMessage_InvalidUTF8BatchDoesNotDropValidMessages(t *testing.T)
require.Equal(t, 3, len(messages))
})
}
func TestStore_AddStats_SumsAcrossNodes(t *testing.T) {
// The global message counter is written by every cluster node; writes must be additive
// increments, not absolute values, or nodes overwrite each other's counts
schemaDSN := dbtest.CreateTestPostgresSchema(t)
openStore := func() *message.Cache {
host, err := pg.Open(schemaDSN)
require.Nil(t, err)
d := db.New(host, nil)
store, err := message.NewPostgresStore(d, 0, 0)
require.Nil(t, err)
t.Cleanup(func() {
store.Close()
})
return store
}
a, b := openStore(), openStore()
require.Nil(t, a.AddStats(5))
require.Nil(t, b.AddStats(3))
require.Nil(t, a.AddStats(2))
messages, err := a.Stats()
require.Nil(t, err)
require.Equal(t, int64(10), messages)
}
func TestStore_CloseGivesUpOnAStuckBatchWrite(t *testing.T) {
// A wedged database must not make Close, and with it the entire shutdown, block forever.
// systemd would SIGKILL us eventually, but only after its stop timeout.
testDB := dbtest.CreateTestPostgres(t)
s, err := message.NewPostgresStore(testDB, 100, time.Hour)
require.Nil(t, err)
// Hold a lock on the message table, so the final batch write cannot make progress
tx, err := testDB.Begin()
require.Nil(t, err)
defer tx.Rollback()
_, err = tx.Exec("LOCK TABLE message IN EXCLUSIVE MODE")
require.Nil(t, err)
require.Nil(t, s.AddMessage(model.NewDefaultMessage("mytopic", "my message")))
done := make(chan struct{})
go func() {
s.Close()
close(done)
}()
select {
case <-done:
case <-time.After(30 * time.Second):
t.Fatal("Close did not return while the database was stuck")
}
}
func TestStore_CloseGivesUpOnAnInFlightBatchWrite(t *testing.T) {
// Same as TestStore_CloseGivesUpOnAStuckBatchWrite, but with a batch already handed to the
// batch writer, so Close cannot even hand over the messages that are still pending
testDB := dbtest.CreateTestPostgres(t)
s, err := message.NewPostgresStore(testDB, 2, time.Hour)
require.Nil(t, err)
tx, err := testDB.Begin()
require.Nil(t, err)
defer tx.Rollback()
_, err = tx.Exec("LOCK TABLE message IN EXCLUSIVE MODE")
require.Nil(t, err)
require.Nil(t, s.AddMessage(model.NewDefaultMessage("mytopic", "stuck 1")))
require.Nil(t, s.AddMessage(model.NewDefaultMessage("mytopic", "stuck 2"))) // Batch of 2, handed over
require.Nil(t, s.AddMessage(model.NewDefaultMessage("mytopic", "pending")))
done := make(chan struct{})
go func() {
s.Close()
close(done)
}()
select {
case <-done:
case <-time.After(30 * time.Second):
t.Fatal("Close did not return while the database was stuck")
}
}
func TestStore_MessagesSinceID_MarkerWrittenAfterALaterMessage(t *testing.T) {
// Cluster nodes write their batches independently, so a message can be written after one
// that was published later. A client whose marker is the late-written one must still get
// the later message, although its row id is lower.
forEachBackend(t, func(t *testing.T, s *message.Cache) {
older := model.NewDefaultMessage("mytopic", "older")
older.Time = 50
sameSecond := model.NewDefaultMessage("mytopic", "same second")
sameSecond.Time = 100
marker := model.NewDefaultMessage("mytopic", "marker")
marker.Time = 100
later := model.NewDefaultMessage("mytopic", "later")
later.Time = 200
require.Nil(t, s.AddMessages([]*model.Message{older, sameSecond, later}))
require.Nil(t, s.AddMessages([]*model.Message{marker})) // Written last, e.g. its node was cut off from the database
messages := messagesAfter(t, s, "mytopic", marker.ID, false)
require.Equal(t, []string{"same second", "later"}, bodies(messages)) // "same second" is the price: the client may have it
})
}
func TestStore_MessagesAfterMarker_ResolvesTheMarkerOnce(t *testing.T) {
// A multi-topic subscription replays every topic after one marker: one lookup of the marker,
// then one query per topic, not a lookup per topic
testDB, proxy := dbtest.CreateTestPostgresWithLatency(t, 0)
s, err := message.NewPostgresStore(testDB, 0, 0)
require.Nil(t, err)
marker := model.NewDefaultMessage("topic0", "marker")
require.Nil(t, s.AddMessages([]*model.Message{marker}))
for i := 0; i < 5; i++ {
require.Nil(t, s.AddMessages([]*model.Message{model.NewDefaultMessage(fmt.Sprintf("topic%d", i), "after")}))
}
resolved, err := s.ResolveMarker(marker.ID)
require.Nil(t, err)
require.NotNil(t, resolved)
_, _, err = s.MessagesAfterMarker("topic0", resolved, false, 0) // Prepares the statement
require.Nil(t, err)
before := proxy.Requests()
for i := 0; i < 5; i++ {
messages, _, err := s.MessagesAfterMarker(fmt.Sprintf("topic%d", i), resolved, false, 0)
require.Nil(t, err)
require.Equal(t, []string{"after"}, bodies(messages))
}
t.Logf("requests for 5 topics: %d", proxy.Requests()-before)
require.LessOrEqual(t, proxy.Requests()-before, int64(5))
}
func TestStore_ResolveMarker_Unknown(t *testing.T) {
forEachBackend(t, func(t *testing.T, s *message.Cache) {
resolved, err := s.ResolveMarker("unknown00000")
require.Nil(t, err)
require.Nil(t, resolved)
})
}
func TestStore_MessagesCapped_RejectsIDMarker(t *testing.T) {
// A since=<id> marker goes through ResolveMarker and MessagesAfterMarker, so the caller
// decides what an unknown marker means; MessagesCapped does not guess
forEachBackend(t, func(t *testing.T, s *message.Cache) {
m := model.NewDefaultMessage("mytopic", "message")
require.Nil(t, s.AddMessages([]*model.Message{m}))
_, _, err := s.MessagesCapped("mytopic", model.NewSinceID(m.ID), false, 0)
require.Error(t, err)
})
}
// messagesAfter replays the topic after a since=<id> marker the cache knows
func messagesAfter(t *testing.T, s *message.Cache, topic, mid string, scheduled bool) []*model.Message {
t.Helper()
marker, err := s.ResolveMarker(mid)
require.Nil(t, err)
require.NotNil(t, marker)
messages, _, err := s.MessagesAfterMarker(topic, marker, scheduled, 0)
require.Nil(t, err)
return messages
}
func bodies(messages []*model.Message) []string {
out := make([]string, 0, len(messages))
for _, m := range messages {
out = append(out, m.Message)
}
return out
}
func TestStore_MessagesSinceID_ForeignMidActsAsPositionalMarker(t *testing.T) {
// Multi-topic subscriptions apply ONE since=<mid> marker to every subscribed topic, so for
// all but the marker's own topic the mid is foreign; it must act as a positional cut-off
// (everything the marker's row precedes), not as "unknown -> full replay"
s := newTestPostgresStore(t)
bOld := model.NewDefaultMessage("topicb", "b old")
bOld.Time = time.Now().Unix() - 10 // An earlier second: the marker's own second is replayed
require.Nil(t, s.AddMessage(bOld))
marker := model.NewDefaultMessage("topica", "marker a")
require.Nil(t, s.AddMessage(marker))
require.Nil(t, s.AddMessage(model.NewDefaultMessage("topicb", "b new")))
messages := messagesAfter(t, s, "topicb", marker.ID, false)
require.Len(t, messages, 1)
require.Equal(t, "b new", messages[0].Message)
}
func TestStore_MessagesSinceID_ReplicaLagFallsBackToPrimary(t *testing.T) {
// since=<mid> resolution must fall back to the primary when the replica does not know the
// mid yet (publish on node A, client reconnects to node B before replication catches up).
// Without the fallback, the unknown mid resolves to "id > 0" and the client is flooded
// with the topic's whole retained history.
primaryDSN, replicaDSN := dbtest.CreateTestPostgresSchema(t), dbtest.CreateTestPostgresSchema(t)
openStoreOn := func(dsn string, replicas []*db.Host) *message.Cache {
host, err := pg.Open(dsn)
require.Nil(t, err)
d := db.New(host, replicas)
d.MarkReplicasHealthyForTest() // Route ReadOnly() to the lagged replica deterministically
store, err := message.NewPostgresStore(d, 0, 0)
require.Nil(t, err)
t.Cleanup(func() {
store.Close()
})
return store
}
// The "replica" is a separate schema holding only the first 3 messages (fully lagged after
// that); row ids align because both schemas insert in the same order
replicaOnly := openStoreOn(replicaDSN, nil)
primaryOnly := openStoreOn(primaryDSN, nil)
for i := 1; i <= 3; i++ {
m := model.NewDefaultMessage("mytopic", fmt.Sprintf("m%d", i))
m.Time = time.Now().Unix() - 10 // An earlier second: the marker's own second is replayed
require.Nil(t, primaryOnly.AddMessage(m))
require.Nil(t, replicaOnly.AddMessage(m))
}
m4 := model.NewDefaultMessage("mytopic", "m4")
require.Nil(t, primaryOnly.AddMessage(m4)) // Not yet replicated
replicaHost, err := pg.OpenReplica(replicaDSN)
require.Nil(t, err)
combined := openStoreOn(primaryDSN, []*db.Host{replicaHost})
messages := messagesAfter(t, combined, "mytopic", m4.ID, false)
require.Len(t, messages, 0) // Correct cut-off via primary; the lagged replica has nothing newer
// A message after the marker that the replica does not have yet either: a reconnecting
// client must get it from the primary, since it went out live before the client arrived
// and the tail does not resend it
m5 := model.NewDefaultMessage("mytopic", "m5")
require.Nil(t, primaryOnly.AddMessage(m5))
messages = messagesAfter(t, combined, "mytopic", m4.ID, false)
require.Equal(t, []string{"m5"}, bodies(messages))
// A recent time-based replay (the fallback for a marker the cache does not know) reads the
// primary for the same reason; an old one is served by the replica as before
messages, err = combined.Messages("mytopic", model.NewSinceTime(time.Now().Unix()-30), false)
require.Nil(t, err)
require.Equal(t, []string{"m1", "m2", "m3", "m4", "m5"}, bodies(messages))
}
message/tail.go
+168
View File
@@ -0,0 +1,168 @@
package message
import (
"database/sql"
"encoding/json"
"sync"
"sync/atomic"
"time"
"heckel.io/ntfy/v2/model"
)
const (
tailPageSize = 1000 // Rows per query
tailMaxPages = 10 // Queries per read at most, so a read drains a backlog of up to 10,000 rows
tailHoleTimeout = time.Minute // How long a skipped row id is re-checked before it counts as never committed (rolled back)
tailMaxHoles = 10000 // Bounds the skipped row ids re-checked on every call
)
// Tail reads the messages written to the cache in write order (by row id), so a node can find
// what it never received live. Row ids are handed out on insert but only become visible on commit,
// and concurrent writers (cluster nodes) commit out of order: a row id the tail had to skip is a
// hole, and is re-checked until its row shows up or the hole times out.
type Tail struct {
cache *Cache
cursor int64 // Highest row id read so far
holes map[int64]time.Time // Skipped row ids below the cursor, and when they were skipped
lastRead atomic.Int64 // Unix nanoseconds of the last successful read (see LastRead); 0 before the first
done chan struct{} // Closed by Close, to stop the loop calling Next
closed sync.Once // Guards closing done
}
// Next returns the published messages written since the last call, in write order, reading
// pages until it is caught up or has read tailMaxPages. It reads the primary: a replica may lag,
// and what it does not show yet would be skipped as a hole.
func (t *Tail) Next() ([]*model.Message, error) {
defer t.expireHoles()
messages := make([]*model.Message, 0)
for i := 0; i < tailMaxPages; i++ {
page, full, err := t.nextPage()
if err != nil {
if len(messages) > 0 {
t.lastRead.Store(time.Now().UnixNano())
return messages, nil // The pages read so far are done; the failed one is retried next time
}
return nil, err
}
messages = append(messages, page...)
if !full {
break
}
}
t.lastRead.Store(time.Now().UnixNano())
return messages, nil
}
// nextPage reads one page of rows: those after the cursor, plus the holes (row ids skipped
// earlier because they were not committed yet). It returns the published messages among them, and
// true if the page was full, so more may be waiting. For each row:
// - a hole that is now visible is "filled": it is removed from the holes
// - a row after the cursor moves the cursor to it; the ids it jumped over are "skipped", and
// become holes
//
// Example: cursor 103, holes {104}; the page returns rows 104, 105 and 108. Row 104 fills its
// hole, 105 moves the cursor, and 108 moves it again, skipping 106 and 107. Afterwards: cursor
// 108, holes {106, 107}. Nothing changes if the read fails part way, so it is retried whole.
func (t *Tail) nextPage() ([]*model.Message, bool, error) {
holes := make([]int64, 0, len(t.holes))
for id := range t.holes {
holes = append(holes, id)
}
holesJSON, err := json.Marshal(holes) // One parameter for any number of holes, so the statement text never changes
if err != nil {
return nil, false, err
}
rows, err := t.cache.db.Primary().Query(t.cache.queries.selectMessagesAfterRowID, t.cursor, string(holesJSON), tailPageSize)
if err != nil {
return nil, false, err
}
defer rows.Close()
cursor, count := t.cursor, 0 // Applied to t only once the whole page was read
filled, skipped := make([]int64, 0), make([]int64, 0) // Holes now visible; row ids jumped over (new holes)
messages := make([]*model.Message, 0) // Published messages, in row id order
for rows.Next() {
m, rowID, published, err := readTailRow(rows)
if err != nil {
return nil, false, err
}
count++
if _, ok := t.holes[rowID]; ok {
filled = append(filled, rowID)
} else if rowID > cursor {
// Row ids between the cursor and this one are not visible (yet)
for id := cursor + 1; id < rowID && len(skipped) < tailMaxHoles; id++ {
skipped = append(skipped, id)
}
cursor = rowID
}
if published { // Scheduled messages go out through the delayed sender
messages = append(messages, m)
}
}
if err := rows.Err(); err != nil {
return nil, false, err
}
// Only now: a read that fails part way is retried whole, from the same position
now := time.Now()
for _, id := range filled {
delete(t.holes, id)
}
for _, id := range skipped {
if len(t.holes) >= tailMaxHoles {
break
}
t.holes[id] = now
}
t.cursor = cursor
return messages, count == tailPageSize, nil
}
// readTailRow reads a row of selectMessagesAfterRowID: the message, then its row id and whether
// it is published (false for a scheduled message)
func readTailRow(rows *sql.Rows) (*model.Message, int64, bool, error) {
var rowID int64
var published bool
m, err := readMessage(rows, &rowID, &published)
if err != nil {
return nil, 0, false, err
}
return m, rowID, published, nil
}
func (t *Tail) expireHoles() {
for id, skipped := range t.holes {
if time.Since(skipped) > tailHoleTimeout {
delete(t.holes, id)
}
}
}
// LastRead returns when the tail last read the database successfully; zero before the first
// read. Unlike Next, it may be called from any goroutine.
func (t *Tail) LastRead() time.Time {
nanos := t.lastRead.Load()
if nanos == 0 {
return time.Time{}
}
return time.Unix(0, nanos)
}
// Done returns a channel that is closed when the tail is closed, so the loop calling Next can stop
func (t *Tail) Done() <-chan struct{} {
return t.done
}
// Close tells the loop calling Next to stop (see Done); it may be called more than once
func (t *Tail) Close() {
t.closed.Do(func() {
close(t.done)
})
}
// addHoles records the row ids between from and to (both exclusive) as holes
func (t *Tail) addHoles(from, to int64, now time.Time) {
for id := from + 1; id < to && len(t.holes) < tailMaxHoles; id++ {
t.holes[id] = now
}
}
message/tail_test.go
+191
View File
@@ -0,0 +1,191 @@
package message_test
import (
"database/sql"
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
dbtest "heckel.io/ntfy/v2/db/test"
"heckel.io/ntfy/v2/message"
"heckel.io/ntfy/v2/model"
)
func TestTail_ReturnsWhatWasWrittenSinceTheLastCall(t *testing.T) {
forEachBackend(t, func(t *testing.T, s *message.Cache) {
require.Nil(t, s.AddMessages([]*model.Message{model.NewDefaultMessage("mytopic", "before")}))
tail, err := s.Tail()
require.Nil(t, err)
requireTail(t, tail)
require.Nil(t, s.AddMessages([]*model.Message{
model.NewDefaultMessage("mytopic", "one"),
model.NewDefaultMessage("othertopic", "two"),
}))
requireTail(t, tail, "one", "two")
requireTail(t, tail)
require.Nil(t, s.AddMessages([]*model.Message{model.NewDefaultMessage("mytopic", "three")}))
requireTail(t, tail, "three")
})
}
func TestTail_ReadsUntilCaughtUp(t *testing.T) {
// More than a page written since the last read: one read still returns all of it
forEachBackend(t, func(t *testing.T, s *message.Cache) {
tail, err := s.Tail()
require.Nil(t, err)
messages := make([]*model.Message, 0)
for i := 0; i < 2500; i++ {
messages = append(messages, model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i)))
}
require.Nil(t, s.AddMessages(messages))
read, err := tail.Next()
require.Nil(t, err)
require.Equal(t, 2500, len(read))
require.Equal(t, "message 2499", read[2499].Message)
})
}
func TestTail_SkipsScheduledMessages(t *testing.T) {
forEachBackend(t, func(t *testing.T, s *message.Cache) {
tail, err := s.Tail()
require.Nil(t, err)
scheduled := model.NewDefaultMessage("mytopic", "later")
scheduled.Time = time.Now().Add(time.Hour).Unix()
require.Nil(t, s.AddMessages([]*model.Message{scheduled, model.NewDefaultMessage("mytopic", "now")}))
requireTail(t, tail, "now")
})
}
func TestTail_ReturnsRowsCommittedOutOfOrder(t *testing.T) {
// Concurrent writers (cluster nodes) commit in their own time, so a row with a lower id can
// become visible after one with a higher id. The tail must not skip it.
testDB := dbtest.CreateTestPostgres(t)
s, err := message.NewPostgresStore(testDB, 0, 0)
require.Nil(t, err)
tail, err := s.Tail()
require.Nil(t, err)
// Take a row id in a transaction that stays open, like a slow batch on another node
tx, err := testDB.Primary().Begin()
require.Nil(t, err)
t.Cleanup(func() { tx.Rollback() }) // Before the schema is dropped, which would wait for it
var rowID int64
require.Nil(t, tx.QueryRow(`SELECT nextval(pg_get_serial_sequence('message', 'id'))`).Scan(&rowID))
// A later row commits first
require.Nil(t, s.AddMessages([]*model.Message{model.NewDefaultMessage("mytopic", "second")}))
requireTail(t, tail, "second")
// The earlier row commits now
insertMessageRow(t, tx, rowID, "first0000000", "first")
require.Nil(t, tx.Commit())
requireTail(t, tail, "first")
requireTail(t, tail)
}
func TestTail_StartupKeepsRowsNotCommittedYet(t *testing.T) {
// A node starts while another node's write is still uncommitted, below a row that is
// already visible: the tail must still pick it up when it commits
testDB := dbtest.CreateTestPostgres(t)
s, err := message.NewPostgresStore(testDB, 0, 0)
require.Nil(t, err)
require.Nil(t, s.AddMessages([]*model.Message{model.NewDefaultMessage("mytopic", "older")}))
tx, err := testDB.Primary().Begin()
require.Nil(t, err)
t.Cleanup(func() { tx.Rollback() }) // Before the schema is dropped, which would wait for it
var rowID int64
require.Nil(t, tx.QueryRow(`SELECT nextval(pg_get_serial_sequence('message', 'id'))`).Scan(&rowID))
require.Nil(t, s.AddMessages([]*model.Message{model.NewDefaultMessage("mytopic", "already committed")}))
tail, err := s.Tail()
require.Nil(t, err)
insertMessageRow(t, tx, rowID, "pending00000", "committed after startup")
require.Nil(t, tx.Commit())
requireTail(t, tail, "committed after startup")
}
func TestTail_FailedReadIsRetriedWhole(t *testing.T) {
// A read that fails part way (here: a row that cannot be decoded) must not move the tail past
// the rows it read before the failure
testDB := dbtest.CreateTestPostgres(t)
s, err := message.NewPostgresStore(testDB, 0, 0)
require.Nil(t, err)
tail, err := s.Tail()
require.Nil(t, err)
second := model.NewDefaultMessage("mytopic", "second")
require.Nil(t, s.AddMessages([]*model.Message{model.NewDefaultMessage("mytopic", "first"), second}))
_, err = testDB.Exec(`UPDATE message SET actions = 'invalid json' WHERE mid = $1`, second.ID)
require.Nil(t, err)
_, err = tail.Next()
require.Error(t, err)
_, err = testDB.Exec(`UPDATE message SET actions = '' WHERE mid = $1`, second.ID)
require.Nil(t, err)
requireTail(t, tail, "first", "second")
}
func insertMessageRow(t *testing.T, tx *sql.Tx, rowID int64, mid, body string) {
_, err := tx.Exec(`
INSERT INTO message (id, mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user_id, content_type, encoding, published)
VALUES ($1, $2, $2, $3, 'message', 0, 'mytopic', $4, '', 0, '', '', '', '', '', '', 0, 0, '', '', '', '', '', TRUE)
`, rowID, mid, time.Now().Unix(), body)
require.Nil(t, err)
}
func TestTail_LastRead(t *testing.T) {
// When the tail last read the database, for a node deciding whether its subscribers still
// get everything; a failed read does not count
testDB := dbtest.CreateTestPostgres(t)
s, err := message.NewPostgresStore(testDB, 0, 0)
require.Nil(t, err)
tail, err := s.Tail()
require.Nil(t, err)
require.True(t, tail.LastRead().IsZero())
before := time.Now()
requireTail(t, tail)
read := tail.LastRead()
require.False(t, read.Before(before))
second := model.NewDefaultMessage("mytopic", "second")
require.Nil(t, s.AddMessages([]*model.Message{second}))
_, err = testDB.Exec(`UPDATE message SET actions = 'invalid json' WHERE mid = $1`, second.ID)
require.Nil(t, err)
_, err = tail.Next()
require.Error(t, err)
require.Equal(t, read, tail.LastRead())
}
func TestTail_Close(t *testing.T) {
// Done tells the loop calling Next to stop; Close may be called more than once
forEachBackend(t, func(t *testing.T, s *message.Cache) {
tail, err := s.Tail()
require.Nil(t, err)
select {
case <-tail.Done():
t.Fatal("done before Close")
default:
}
tail.Close()
tail.Close()
select {
case <-tail.Done():
case <-time.After(time.Second):
t.Fatal("not done after Close")
}
})
}
func requireTail(t *testing.T, tail *message.Tail, expected ...string) {
t.Helper()
messages, err := tail.Next()
require.Nil(t, err)
actual := make([]string, 0)
for _, m := range messages {
actual = append(actual, m.Message)
}
require.Equal(t, append([]string{}, expected...), actual)
}
metrics/metrics.go
+138
View File
@@ -4,6 +4,9 @@
package metrics
import (
"database/sql"
"sync/atomic"
"github.com/prometheus/client_golang/prometheus"
)
@@ -75,6 +78,46 @@ var (
HTTPRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "ntfy_http_requests_total",
}, []string{"http_code", "ntfy_code", "http_method"})
ClusterPeers = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "ntfy_cluster_peers",
})
ClusterMessagesForwarded = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_cluster_messages_forwarded_total",
})
ClusterBatchesSent = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_cluster_batches_sent_total",
})
ClusterSendErrors = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_cluster_send_errors_total",
})
ClusterQueueDropped = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_cluster_queue_dropped_total",
})
ClusterMessagesWasted = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_cluster_messages_wasted_total",
})
ClusterLeader = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "ntfy_cluster_leader",
})
ClusterPeerBatchesSent = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "ntfy_cluster_peer_batches_sent_total", // Per peer, result "ok" or "error": a link that only fails shows up here
}, []string{"peer", "result"})
ClusterPeerBatchesReceived = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "ntfy_cluster_peer_batches_received_total",
}, []string{"peer"})
ClusterPeerLastSendSuccess = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "ntfy_cluster_peer_last_send_success_timestamp_seconds",
}, []string{"peer"})
TailMessagesLagSeconds = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "ntfy_tail_messages_lag_seconds", // Age of the newest message of the last read; well above the write delay means it is behind
})
TailMessagesRead = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_tail_messages_read_total",
})
TailMessagesMissed = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_tail_messages_missed_total", // Sent by the tail, per subscriber that had missed it
})
dbPools = newDBPoolCollector() // Read through SetDBPoolsFunc
)
// init registers all collectors with the default Prometheus registry. Registration is
@@ -98,10 +141,105 @@ func init() {
MatrixPublishedSuccess,
MatrixPublishedFailure,
AttachmentsTotalSize,
ClusterPeers,
ClusterMessagesForwarded,
ClusterBatchesSent,
ClusterSendErrors,
ClusterQueueDropped,
ClusterMessagesWasted,
ClusterLeader,
ClusterPeerBatchesSent,
ClusterPeerBatchesReceived,
ClusterPeerLastSendSuccess,
TailMessagesLagSeconds,
TailMessagesRead,
TailMessagesMissed,
Visitors,
Users,
Subscribers,
Topics,
HTTPRequests,
dbPools,
)
}
// DBPool is one database connection pool's statistics: the primary's or a replica's
type DBPool struct {
Role string // db.PoolRolePrimary or db.PoolRoleReplica
Addr string // "host:port"
Stats sql.DBStats
}
// SetDBPoolsFunc sets the function that the ntfy_db_pool_* metrics read their values from on
// every scrape; nil (the default) reports no pools
func SetDBPoolsFunc(fn func() []*DBPool) {
dbPools.fn.Store(&fn)
}
// dbPoolCollector exports the database connection pools' statistics. It reads them at scrape
// time rather than on a timer, so the in-use count is as fresh as the scrape, and a burst that
// exhausts a pool between two scrapes still shows up in the wait counters.
type dbPoolCollector struct {
fn atomic.Pointer[func() []*DBPool]
metrics []*dbPoolMetric
}
// dbPoolMetric is one ntfy_db_pool_* metric: its description, and how to read it from a pool
type dbPoolMetric struct {
desc *prometheus.Desc
valueType prometheus.ValueType
value func(stats sql.DBStats) float64
}
func newDBPoolCollector() *dbPoolCollector {
return &dbPoolCollector{
metrics: []*dbPoolMetric{
newDBPoolMetric("ntfy_db_pool_connections_max", "Maximum open database connections (pool_max_conns), per pool", prometheus.GaugeValue, func(stats sql.DBStats) float64 {
return float64(stats.MaxOpenConnections)
}),
newDBPoolMetric("ntfy_db_pool_connections_open", "Open database connections, in use or idle, per pool", prometheus.GaugeValue, func(stats sql.DBStats) float64 {
return float64(stats.OpenConnections)
}),
newDBPoolMetric("ntfy_db_pool_connections_in_use", "Database connections currently in use, per pool", prometheus.GaugeValue, func(stats sql.DBStats) float64 {
return float64(stats.InUse)
}),
newDBPoolMetric("ntfy_db_pool_connections_idle", "Idle database connections, per pool", prometheus.GaugeValue, func(stats sql.DBStats) float64 {
return float64(stats.Idle)
}),
newDBPoolMetric("ntfy_db_pool_waits_total", "Times a query had to wait for a free database connection, per pool", prometheus.CounterValue, func(stats sql.DBStats) float64 {
return float64(stats.WaitCount)
}),
newDBPoolMetric("ntfy_db_pool_wait_seconds_total", "Total time spent waiting for a free database connection, per pool", prometheus.CounterValue, func(stats sql.DBStats) float64 {
return stats.WaitDuration.Seconds()
}),
},
}
}
func newDBPoolMetric(name, help string, valueType prometheus.ValueType, value func(stats sql.DBStats) float64) *dbPoolMetric {
return &dbPoolMetric{
desc: prometheus.NewDesc(name, help, []string{"role", "host"}, nil),
valueType: valueType,
value: value,
}
}
// Describe implements prometheus.Collector
func (c *dbPoolCollector) Describe(ch chan<- *prometheus.Desc) {
for _, m := range c.metrics {
ch <- m.desc
}
}
// Collect implements prometheus.Collector
func (c *dbPoolCollector) Collect(ch chan<- prometheus.Metric) {
fn := c.fn.Load()
if fn == nil || *fn == nil {
return
}
for _, pool := range (*fn)() {
for _, m := range c.metrics {
ch <- prometheus.MustNewConstMetric(m.desc, m.valueType, m.value(pool.Stats), pool.Role, pool.Addr)
}
}
}
metrics/metrics_test.go
+98 -1
View File
@@ -1,12 +1,17 @@
package metrics
import (
"database/sql"
"fmt"
"sort"
"strings"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/require"
"heckel.io/ntfy/v2/db"
)
// expectedMetricNames is the exact set of metrics the server exposes. These names are a public
@@ -15,6 +20,22 @@ var expectedMetricNames = []string{
"ntfy_attachments_total_size",
"ntfy_calls_made_failure",
"ntfy_calls_made_success",
"ntfy_cluster_batches_sent_total",
"ntfy_cluster_leader",
"ntfy_cluster_messages_forwarded_total",
"ntfy_cluster_messages_wasted_total",
"ntfy_cluster_peer_batches_received_total",
"ntfy_cluster_peer_batches_sent_total",
"ntfy_cluster_peer_last_send_success_timestamp_seconds",
"ntfy_cluster_peers",
"ntfy_cluster_queue_dropped_total",
"ntfy_cluster_send_errors_total",
"ntfy_db_pool_connections_idle",
"ntfy_db_pool_connections_in_use",
"ntfy_db_pool_connections_max",
"ntfy_db_pool_connections_open",
"ntfy_db_pool_wait_seconds_total",
"ntfy_db_pool_waits_total",
"ntfy_emails_received_failure",
"ntfy_emails_received_success",
"ntfy_emails_sent_failure",
@@ -29,6 +50,9 @@ var expectedMetricNames = []string{
"ntfy_messages_published_failure",
"ntfy_messages_published_success",
"ntfy_subscribers_total",
"ntfy_tail_messages_lag_seconds",
"ntfy_tail_messages_missed_total",
"ntfy_tail_messages_read_total",
"ntfy_topics_total",
"ntfy_unifiedpush_published_success",
"ntfy_users_total",
@@ -36,7 +60,19 @@ var expectedMetricNames = []string{
}
func TestRegisteredMetricNames(t *testing.T) {
HTTPRequests.WithLabelValues("200", "20000", "GET").Inc()
HTTPRequests.WithLabelValues("200", "20000", "GET").Inc() // Vectors are only gathered once they have a series
ClusterPeerBatchesSent.WithLabelValues("node-b", "ok").Inc()
ClusterPeerBatchesReceived.WithLabelValues("node-b").Inc()
ClusterPeerLastSendSuccess.WithLabelValues("node-b").Set(1)
SetDBPoolsFunc(func() []*DBPool {
return []*DBPool{
{
Role: db.PoolRolePrimary,
Addr: "db:5432",
},
}
})
defer SetDBPoolsFunc(nil)
families, err := prometheus.DefaultGatherer.Gather()
require.Nil(t, err)
names := make([]string, 0)
@@ -56,3 +92,64 @@ func TestCollectors_NeverNil(t *testing.T) {
MessagesCached.Set(1)
HTTPRequests.WithLabelValues("200", "20000", "PUT").Inc()
}
func TestDBPoolMetrics(t *testing.T) {
// Read from the pools on every scrape, one series per pool, so a burst that exhausts a pool
// shows up as waits even between two scrapes
SetDBPoolsFunc(func() []*DBPool {
return []*DBPool{
{
Role: db.PoolRolePrimary,
Addr: "db:5432",
Stats: sql.DBStats{
MaxOpenConnections: 30,
OpenConnections: 12,
InUse: 5,
Idle: 7,
WaitCount: 3,
WaitDuration: 1500 * time.Millisecond,
},
},
{
Role: db.PoolRoleReplica,
Addr: "replica:5432",
Stats: sql.DBStats{
MaxOpenConnections: 20,
OpenConnections: 2,
InUse: 1,
Idle: 1,
},
},
}
})
defer SetDBPoolsFunc(nil)
expected := fmt.Sprintf(`
# HELP ntfy_db_pool_connections_in_use Database connections currently in use, per pool
# TYPE ntfy_db_pool_connections_in_use gauge
ntfy_db_pool_connections_in_use{host="db:5432",role="%[1]s"} 5
ntfy_db_pool_connections_in_use{host="replica:5432",role="%[2]s"} 1
# HELP ntfy_db_pool_connections_max Maximum open database connections (pool_max_conns), per pool
# TYPE ntfy_db_pool_connections_max gauge
ntfy_db_pool_connections_max{host="db:5432",role="%[1]s"} 30
ntfy_db_pool_connections_max{host="replica:5432",role="%[2]s"} 20
# HELP ntfy_db_pool_wait_seconds_total Total time spent waiting for a free database connection, per pool
# TYPE ntfy_db_pool_wait_seconds_total counter
ntfy_db_pool_wait_seconds_total{host="db:5432",role="%[1]s"} 1.5
ntfy_db_pool_wait_seconds_total{host="replica:5432",role="%[2]s"} 0
# HELP ntfy_db_pool_waits_total Times a query had to wait for a free database connection, per pool
# TYPE ntfy_db_pool_waits_total counter
ntfy_db_pool_waits_total{host="db:5432",role="%[1]s"} 3
ntfy_db_pool_waits_total{host="replica:5432",role="%[2]s"} 0
`, db.PoolRolePrimary, db.PoolRoleReplica)
require.Nil(t, testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected),
"ntfy_db_pool_connections_in_use", "ntfy_db_pool_connections_max", "ntfy_db_pool_waits_total", "ntfy_db_pool_wait_seconds_total"))
open, err := testutil.GatherAndCount(prometheus.DefaultGatherer, "ntfy_db_pool_connections_open", "ntfy_db_pool_connections_idle")
require.Nil(t, err)
require.Equal(t, 4, open)
// Without a database (SQLite, or before the server sets it), there are no series
SetDBPoolsFunc(nil)
count, err := testutil.GatherAndCount(prometheus.DefaultGatherer, "ntfy_db_pool_connections_max")
require.Nil(t, err)
require.Equal(t, 0, count)
}
mkdocs.yml
+1
View File
@@ -105,6 +105,7 @@ nav:
- "Deprecation notices": deprecations.md
- "Development": develop.md
- "Contributing": contributing.md
- "AI policy": ai-policy.md
- "Privacy policy": privacy.md
- "Terms of Service": terms.md
- "Contact": contact.md
scripts/check-cpu-baseline.sh
Executable
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
# Runs an amd64 binary on an emulated baseline x86-64 CPU (qemu64: no AVX2, BMI2, MOVBE), to catch
# statically linked C code (glibc, SQLite) that was built for newer CPUs, see #2092.
#
# Fails without qemu-x86_64 in CI (CI is set by GitHub Actions), skips with a warning elsewhere.
set -euo pipefail
binary="$1"
if ! command -v qemu-x86_64 >/dev/null; then
if [ -n "${CI:-}" ]; then
echo "ERROR: qemu-x86_64 not found, cannot check $binary on a baseline x86-64 CPU. On Ubuntu, run: apt install qemu-user"
exit 1
fi
echo "WARNING: qemu-x86_64 not found, skipping baseline x86-64 CPU check of $binary"
exit 0
fi
if ! qemu-x86_64 -cpu qemu64 "$binary" --help >/dev/null; then
echo "ERROR: $binary does not run on a baseline x86-64 CPU (qemu64), it likely contains x86-64-v3 instructions (see #2092)"
exit 1
fi
echo "$binary runs on a baseline x86-64 CPU (qemu64)"
server/config.go
+29 -8
View File
@@ -11,6 +11,7 @@ import (
"time"
"heckel.io/ntfy/v2/ban"
"heckel.io/ntfy/v2/cluster"
"heckel.io/ntfy/v2/user"
)
@@ -25,10 +26,14 @@ const (
DefaultDelayedSenderInterval = 10 * time.Second
DefaultMessageDelayMin = 10 * time.Second
DefaultMessageDelayMax = 3 * 24 * time.Hour
DefaultFirebaseKeepaliveInterval = 3 * time.Hour // ~control topic (Android), not too frequently to save battery
DefaultFirebasePollInterval = 20 * time.Minute // ~poll topic (iOS), max. 2-3 times per hour (see docs)
DefaultFirebaseQuotaExceededPenaltyDuration = 10 * time.Minute // Time that over-users are locked out of Firebase if it returns "quota exceeded"
DefaultStripePriceCacheDuration = 3 * time.Hour // Time to keep Stripe prices cached in memory before a refresh is needed
DefaultFirebaseKeepaliveInterval = 3 * time.Hour // ~control topic (Android), not too frequently to save battery
DefaultFirebasePollInterval = 20 * time.Minute // ~poll topic (iOS), max. 2-3 times per hour (see docs)
DefaultFirebaseQuotaExceededPenaltyDuration = 10 * time.Minute // Time that over-users are locked out of Firebase if it returns "quota exceeded"
DefaultStripePriceCacheDuration = 3 * time.Hour // Time to keep Stripe prices cached in memory before a refresh is needed
DefaultClusterBatchLinger = 500 * time.Millisecond // How long fan-out messages wait to form a per-peer batch (experimental clustering)
DefaultClusterRole = cluster.RoleActive // Active nodes take client traffic, lead and fence (experimental clustering)
DefaultClusterDrainDelay = 15 * time.Second // On shutdown, how long a draining node withdraws (not ready, still serves everything); with the drain duration, fits systemd's default stop timeout (90s) (experimental clustering)
DefaultClusterDrainDuration = time.Minute // On shutdown, how long a draining node takes to close its streams, read-only (experimental clustering)
)
// Platform-specific default paths (set in config_unix.go or config_windows.go)
@@ -123,14 +128,22 @@ var (
type Config struct {
File string // Config file, only used for testing
BaseURL string
ListenHTTP string
ListenHTTP []string // Several addresses let a load balancer open more than ~63k connections to this node
ListenHTTPS string
ListenUnix string
ListenUnixMode fs.FileMode
KeyFile string
CertFile string
DatabaseURL string // PostgreSQL connection string (e.g. "postgres://user:pass@host:5432/ntfy")
DatabaseReplicaURLs []string // PostgreSQL read replica connection strings
DatabaseURL string // PostgreSQL connection string (e.g. "postgres://user:pass@host:5432/ntfy")
DatabaseReplicaURLs []string // PostgreSQL read replica connection strings
ClusterNodeID string // Stable per-node identifier used to skip a node's own fan-out; required in cluster mode
ClusterListen string // ip:port the dedicated cluster fan-out listener binds to (private network, e.g. "10.0.0.5:2587")
ClusterAdvertiseURL string // Base URL peers use to reach this node's fan-out listener (defaults to "http://<cluster-listen>")
ClusterSecret string `hash:"-"` // Shared secret authenticating node-to-node fan-out requests
ClusterBatchLinger time.Duration // How long fan-out messages wait to form a batch per peer; 0 sends immediately
ClusterRole cluster.Role // Active (default) or passive: a passive node never leads and never fences
ClusterDrainDelay time.Duration // On shutdown, how long a draining node withdraws (not ready, still serves everything) before it turns read-only
ClusterDrainDuration time.Duration // On shutdown, how long an active node takes to close its streams so they reconnect to other nodes; 0 disables
FirebaseKeyFile string
CacheFile string
CacheDuration time.Duration
@@ -240,13 +253,21 @@ func NewConfig() *Config {
return &Config{
File: DefaultConfigFile, // Only used for testing
BaseURL: "",
ListenHTTP: DefaultListenHTTP,
ListenHTTP: []string{DefaultListenHTTP},
ListenHTTPS: "",
ListenUnix: "",
ListenUnixMode: 0,
KeyFile: "",
CertFile: "",
DatabaseURL: "",
ClusterNodeID: "",
ClusterListen: "",
ClusterAdvertiseURL: "",
ClusterSecret: "",
ClusterBatchLinger: DefaultClusterBatchLinger,
ClusterRole: DefaultClusterRole,
ClusterDrainDelay: DefaultClusterDrainDelay,
ClusterDrainDuration: DefaultClusterDrainDuration,
FirebaseKeyFile: "",
CacheFile: "",
CacheDuration: DefaultCacheDuration,
server/config_test.go
+1 -1
View File
@@ -9,7 +9,7 @@ import (
func TestConfig_New(t *testing.T) {
c := server.NewConfig()
assert.Equal(t, ":80", c.ListenHTTP)
assert.Equal(t, []string{server.DefaultListenHTTP}, c.ListenHTTP)
assert.Equal(t, server.DefaultKeepaliveInterval, c.KeepaliveInterval)
}
server/errors.go
+15
View File
@@ -151,9 +151,21 @@ var (
errHTTPBadRequestTemplateTooLarge = &errHTTP{40056, http.StatusBadRequest, "invalid request: template too large", "https://ntfy.sh/docs/publish/#message-templating", nil}
errHTTPBadRequestTitleTooLarge = &errHTTP{40057, http.StatusBadRequest, "invalid request: title is too large", "https://ntfy.sh/docs/publish/#limitations", nil}
errHTTPBadRequestTagsTooLarge = &errHTTP{40058, http.StatusBadRequest, "invalid request: tags are too large", "https://ntfy.sh/docs/publish/#limitations", nil}
errHTTPBadRequestTimezoneInvalid = &errHTTP{40059, http.StatusBadRequest, "invalid request: invalid timezone", "https://ntfy.sh/docs/publish/#scheduled-delivery", nil}
errHTTPBadRequestPhoneVerificationCodeInvalid = &errHTTP{40060, http.StatusBadRequest, "invalid request: phone number verification code invalid", "", nil}
errHTTPBadRequestTooManyTopics = &errHTTP{40061, http.StatusBadRequest, "invalid request: too many topics", "https://ntfy.sh/docs/subscribe/api/#subscribe-to-multiple-topics", nil}
errHTTPBadRequestTierCodeInvalid = &errHTTP{40062, http.StatusBadRequest, "invalid request: tier code invalid, or tier name missing", "", nil}
errHTTPBadRequestTierReservationsLimit = &errHTTP{40063, http.StatusBadRequest, "invalid request: user has more topic reservations than the tier allows, remove reservations first", "", nil}
errHTTPBadRequestAccessAdminUser = &errHTTP{40064, http.StatusBadRequest, "invalid request: admins can access all topics, access control entries are not allowed", "", nil}
errHTTPBadRequestTokenNotFound = &errHTTP{40065, http.StatusBadRequest, "invalid request: token does not exist", "", nil}
errHTTPBadRequestRoleInvalid = &errHTTP{40066, http.StatusBadRequest, "invalid request: role must be user or admin", "", nil}
errHTTPBadRequestUserNoPrimaryEmail = &errHTTP{40067, http.StatusBadRequest, "invalid request: user has no primary email, cannot send reset link", "", nil}
errHTTPBadRequestBaseURLNotConfigured = &errHTTP{40068, http.StatusBadRequest, "invalid request: base-url must be configured on the server to generate links", "https://ntfy.sh/docs/config/", nil}
errHTTPBadRequestPasswordHashInvalid = &errHTTP{40069, http.StatusBadRequest, "invalid request: password hash must be a bcrypt hash with cost 10 or higher, use 'ntfy user hash' to generate", "", nil}
errHTTPNotFound = &errHTTP{40401, http.StatusNotFound, "page not found", "", nil}
errHTTPUnauthorized = &errHTTP{40101, http.StatusUnauthorized, "unauthorized", "https://ntfy.sh/docs/publish/#authentication", nil}
errHTTPForbidden = &errHTTP{40301, http.StatusForbidden, "forbidden", "https://ntfy.sh/docs/publish/#authentication", nil}
errHTTPForbiddenAdminSocketOnly = &errHTTP{40302, http.StatusForbidden, "forbidden: this operation is only allowed over the local admin socket", "https://ntfy.sh/docs/config/#admin-socket", nil}
errHTTPConflictUserExists = &errHTTP{40901, http.StatusConflict, "conflict: user already exists", "", nil}
errHTTPConflictTopicReserved = &errHTTP{40902, http.StatusConflict, "conflict: access control entry for topic or topic pattern already exists", "", nil}
errHTTPConflictSubscriptionExists = &errHTTP{40903, http.StatusConflict, "conflict: topic subscription already exists", "", nil}
@@ -162,6 +174,8 @@ var (
errHTTPConflictProvisionedTokenChange = &errHTTP{40906, http.StatusConflict, "conflict: cannot change or delete provisioned token", "", nil}
errHTTPConflictEmailExists = &errHTTP{40907, http.StatusConflict, "conflict: email address already exists", "", nil}
errHTTPConflictEmailPrimaryElsewhere = &errHTTP{40908, http.StatusConflict, "conflict: email address is the primary email on another account", "", nil}
errHTTPConflictTierExists = &errHTTP{40909, http.StatusConflict, "conflict: tier already exists", "", nil}
errHTTPConflictTierInUse = &errHTTP{40910, http.StatusConflict, "conflict: tier is still assigned to users", "", nil}
errHTTPGonePhoneVerificationExpired = &errHTTP{41001, http.StatusGone, "phone number verification expired or does not exist", "", nil}
errHTTPEntityTooLargeAttachment = &errHTTP{41301, http.StatusRequestEntityTooLarge, "attachment too large, or bandwidth limit reached", "https://ntfy.sh/docs/publish/#limitations", nil}
errHTTPEntityTooLargeMatrixRequest = &errHTTP{41302, http.StatusRequestEntityTooLarge, "Matrix request is larger than the max allowed length", "", nil}
@@ -181,5 +195,6 @@ var (
errHTTPInternalErrorInvalidPath = &errHTTP{50002, http.StatusInternalServerError, "internal server error: invalid path", "", nil}
errHTTPInternalErrorMissingBaseURL = &errHTTP{50003, http.StatusInternalServerError, "internal server error: base-url must be be configured for this feature", "https://ntfy.sh/docs/config/", nil}
errHTTPInternalErrorWebPushUnableToPublish = &errHTTP{50004, http.StatusInternalServerError, "internal server error: unable to publish web push message", "", nil}
errHTTPServiceUnavailableDraining = &errHTTP{50301, http.StatusServiceUnavailable, "service unavailable: this node is shutting down, please reconnect", "", nil}
errHTTPInsufficientStorageUnifiedPush = &errHTTP{50701, http.StatusInsufficientStorage, "cannot publish to UnifiedPush topic without previously active subscriber", "", nil}
)
server/log.go
+1
View File
@@ -31,6 +31,7 @@ const (
tagWebsocket = "websocket"
tagMatrix = "matrix"
tagWebPush = "webpush"
tagAdmin = "admin"
)
var (
server/server.go
+334 -86
View File
@@ -22,6 +22,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"unicode/utf8"
@@ -32,8 +33,10 @@ import (
"heckel.io/ntfy/v2/action"
"heckel.io/ntfy/v2/attachment"
"heckel.io/ntfy/v2/ban"
"heckel.io/ntfy/v2/cluster"
"heckel.io/ntfy/v2/db"
"heckel.io/ntfy/v2/db/pg"
"heckel.io/ntfy/v2/limit"
"heckel.io/ntfy/v2/log"
"heckel.io/ntfy/v2/mail"
"heckel.io/ntfy/v2/message"
@@ -49,22 +52,26 @@ import (
// Server is the main server, providing the UI and API for ntfy
type Server struct {
config *Config
db *db.DB // Shared PostgreSQL connection pool (with optional replicas), nil when using SQLite
httpServer *http.Server
db *db.DB // Shared PostgreSQL connection pool (with optional replicas), nil when using SQLite
cluster cluster.Cluster // Fans messages out to peer cluster nodes (nop when not clustered)
httpServers []*http.Server
httpsServer *http.Server
httpMetricsServer *http.Server
httpProfileServer *http.Server
httpClusterServer *http.Server // Dedicated private listener for node-to-node fan-out (experimental-cluster-listen)
unixListener net.Listener
smtpServer *smtp.Server
smtpServerBackend *smtpBackend
mailer mail.Sender
topics map[string]*topic
visitors map[string]*visitor // ip:<ip> or user:<user>
ban *ban.Service // Abuse ban-feed; nil when the feature is disabled (no ban file)
limiter *limit.Service // Decides on and counts the visitors' usage, keyed by visitor key
visitors *visitorEnv // Shared by all visitors, which are built per request
ban *ban.Service // Abuse ban-feed; nil when the feature is disabled (no ban file)
firebaseClient *firebaseClient
twilio *twilio.Client
messages int64 // Total number of messages (persisted if messageCache enabled)
messagesHistory []int64 // Last n values of the messages counter, used to determine rate
messagesFlushed int64 // Value of the messages counter at the last stats flush; the difference is written as a delta
userManager *user.Manager // Might be nil!
messageCache *message.Cache // Database that stores the messages
webPush *webpush.Store // Database that stores web push subscriptions
@@ -73,6 +80,11 @@ type Server struct {
priceCache *util.LookupCache[map[string]int64] // Stripe price ID -> price as cents (USD implied!)
metricsHandler http.Handler // Handles /metrics if enable-metrics set, and listen-metrics-http not set
closeChan chan bool
tail *message.Tail // Reads what was written, see runTail; nil without a message cache
stopOnce sync.Once
stopped atomic.Bool // Set by Stop; read by Run, which must not start listeners afterwards
streams *streamRegistry // Open subscriber connections, for Drain
readOnly atomic.Bool // Second phase of a drain: new subscriptions and publishes get 503 (see Drain)
mu sync.RWMutex
}
@@ -106,6 +118,7 @@ var (
matrixPushPath = "/_matrix/push/v1/notify"
metricsPath = "/metrics"
apiHealthPath = "/v1/health"
apiReadyPath = "/v1/ready"
apiVersionPath = "/v1/version"
apiConfigPath = "/v1/config"
apiStatsPath = "/v1/stats"
@@ -113,6 +126,8 @@ var (
apiTiersPath = "/v1/tiers"
apiUsersPath = "/v1/users"
apiUsersAccessPath = "/v1/users/access"
apiUsersTokensPath = "/v1/users/tokens"
apiUsersPasswordResetPath = "/v1/users/password/reset"
apiAccountPath = "/v1/account"
apiAccountLoginPath = "/v1/account/login"
apiAccountTokenPath = "/v1/account/token"
@@ -128,6 +143,7 @@ var (
apiAccountEmailResendPath = "/v1/account/email/resend"
apiAccountPasswordResetRequestPath = "/v1/account/password/reset/request"
apiAccountPasswordResetPath = "/v1/account/password/reset"
apiAccountBillingTiersPath = "/v1/account/billing/tiers"
apiAccountBillingPortalPath = "/v1/account/billing/portal"
apiAccountBillingWebhookPath = "/v1/account/billing/webhook"
apiAccountBillingSubscriptionPath = "/v1/account/billing/subscription"
@@ -163,6 +179,12 @@ const (
unifiedPushTopicPrefix = "up" // Temporarily, we rate limit all "up*" topics based on the subscriber
unifiedPushTopicLength = 14 // Length of UnifiedPush topics, including the "up" part
messagesHistoryMax = 10 // Number of message count values to keep in memory
subscribeTopicsLimit = 150 // Max distinct topics in one subscription; each one runs its own replay query (matches the ntfy.sh nginx edge rule)
// stopTimeout bounds the entire shutdown. The stores wait for their own background work
// (the attachment sync loop queries the database), and none of those waits has a deadline,
// so this is the backstop that keeps a wedged database from stalling us until SIGKILL.
stopTimeout = 10 * time.Second
)
// WebSocket constants
@@ -253,6 +275,40 @@ func New(conf *Config) (*Server, error) {
CallFormat: conf.TwilioCallFormat,
BuildVersion: conf.BuildVersion,
})
var banner *ban.Service
if conf.BanFile != "" {
banner = ban.NewService(&ban.Config{
File: conf.BanFile,
Window: conf.BanWindow,
Threshold: conf.BanThreshold,
Weights: conf.BanWeights,
PrefixBitsIPv4: conf.VisitorPrefixBitsIPv4,
PrefixBitsIPv6: conf.VisitorPrefixBitsIPv6,
})
}
// Peers reach this node on the dedicated cluster listener, never on the public ones
advertiseURL := conf.ClusterAdvertiseURL
if advertiseURL == "" && conf.ClusterListen != "" {
advertiseURL = "http://" + conf.ClusterListen
}
s := &Server{
config: conf,
db: pool,
messageCache: messageCache,
webPush: wp,
attachment: attachmentStore,
twilio: twilioClient,
mailer: sender,
ban: banner,
topics: topics,
messages: messages,
messagesHistory: []int64{messages},
messagesFlushed: messages, // The loaded total is already persisted; only new publishes are deltas
limiter: limit.New(&limit.Config{ResetTime: conf.VisitorStatsResetTime}),
stripe: stripe,
streams: newStreamRegistry(),
}
s.priceCache = util.NewLookupCache(s.fetchStripePrices, conf.StripePriceCacheDuration)
var userManager *user.Manager
if conf.AuthFile != "" || pool != nil {
authConfig := &user.Config{
@@ -268,6 +324,8 @@ func New(conf *Config) (*Server, error) {
QueueWriterInterval: conf.AuthStatsQueueWriterInterval,
AccessCacheEnabled: conf.AuthAccessCacheEnabled,
AccessCacheReloadInterval: conf.AuthAccessCacheReloadInterval,
AccessCacheChangedFunc: s.reauthorizeSubscribers,
AccessWrittenFunc: s.broadcastAccessChanged,
}
if pool != nil {
userManager, err = user.NewPostgresManager(pool, authConfig)
@@ -292,36 +350,49 @@ func New(conf *Config) (*Server, error) {
}
firebaseClient = newFirebaseClient(sender, auther)
}
var banner *ban.Service
if conf.BanFile != "" {
banner = ban.NewService(&ban.Config{
File: conf.BanFile,
Window: conf.BanWindow,
Threshold: conf.BanThreshold,
Weights: conf.BanWeights,
PrefixBitsIPv4: conf.VisitorPrefixBitsIPv4,
PrefixBitsIPv6: conf.VisitorPrefixBitsIPv6,
s.userManager = userManager
s.visitors = newVisitorEnv(conf, s.limiter, messageCache, userManager)
s.firebaseClient = firebaseClient
if pool != nil {
metrics.SetDBPoolsFunc(s.dbPoolStats)
}
s.cluster, err = cluster.New(&cluster.Config{
Enabled: conf.ClusterListen != "", // Setting experimental-cluster-listen implicitly enables clustering
NodeID: cluster.NodeID(conf.ClusterNodeID),
AdvertiseURL: advertiseURL,
Role: conf.ClusterRole,
Secret: conf.ClusterSecret,
BatchLinger: conf.ClusterBatchLinger,
UsersDeletedFunc: s.handleUsersDeleted,
AccessChangedFunc: s.handlePeerAccessChanged,
IsolatedFunc: s.maybeCloseLocalSubscribers,
MaxMessageBytes: int64(conf.MessageSizeLimit)*4 + 1024, // Envelope overhead over the raw message
}, pool, s.deliverFromBus)
if err != nil {
return nil, err
}
if conf.CacheDuration > 0 {
tail, err := messageCache.Tail()
if err != nil {
return nil, err
}
s.tail = tail
go s.runTail()
}
return s, nil
}
// dbPoolStats reports the database connection pools for the ntfy_db_pool_* metrics
func (s *Server) dbPoolStats() []*metrics.DBPool {
pools := make([]*metrics.DBPool, 0)
for _, p := range s.db.Stats() {
pools = append(pools, &metrics.DBPool{
Role: p.Role,
Addr: p.Addr,
Stats: p.Stats,
})
}
s := &Server{
config: conf,
db: pool,
messageCache: messageCache,
webPush: wp,
attachment: attachmentStore,
firebaseClient: firebaseClient,
twilio: twilioClient,
mailer: sender,
ban: banner,
topics: topics,
userManager: userManager,
messages: messages,
messagesHistory: []int64{messages},
visitors: make(map[string]*visitor),
stripe: stripe,
}
s.priceCache = util.NewLookupCache(s.fetchStripePrices, conf.StripePriceCacheDuration)
return s, nil
return pools
}
func createMessageCache(conf *Config, pool *db.DB) (*message.Cache, error) {
@@ -344,12 +415,17 @@ func createAttachmentStore(conf *Config, messageCache *message.Cache) (*attachme
return nil, nil
}
// testHookBeforeSubscribe, if non-nil, is called by the subscribe handlers after the request was
// authorized and before its stream subscribes. Tests use it to land an access change in between;
// it is always nil in production.
var testHookBeforeSubscribe func()
// Run executes the main server. It listens on HTTP (+ HTTPS, if configured), and starts
// a manager go routine to print stats and prune messages.
func (s *Server) Run() error {
var listenStr string
if s.config.ListenHTTP != "" {
listenStr += fmt.Sprintf(" %s[http]", s.config.ListenHTTP)
for _, addr := range s.config.ListenHTTP {
listenStr += fmt.Sprintf(" %s[http]", addr)
}
if s.config.ListenHTTPS != "" {
listenStr += fmt.Sprintf(" %s[https]", s.config.ListenHTTPS)
@@ -374,12 +450,20 @@ func (s *Server) Run() error {
mux := http.NewServeMux()
mux.HandleFunc("/", s.handle)
errChan := make(chan error)
if s.stopped.Load() {
return nil // Stopped before we got here; a stuck shutdown must not block us on the lock
}
s.mu.Lock()
if s.stopped.Load() {
s.mu.Unlock()
return nil // Stopped while we waited for the lock, i.e. the stores are closed by now
}
s.closeChan = make(chan bool)
if s.config.ListenHTTP != "" {
s.httpServer = &http.Server{Addr: s.config.ListenHTTP, Handler: mux}
for _, addr := range s.config.ListenHTTP {
httpServer := &http.Server{Addr: addr, Handler: mux}
s.httpServers = append(s.httpServers, httpServer)
go func() {
errChan <- s.httpServer.ListenAndServe()
errChan <- httpServer.ListenAndServe()
}()
}
if s.config.ListenHTTPS != "" {
@@ -392,6 +476,10 @@ func (s *Server) Run() error {
go func() {
var err error
s.mu.Lock()
if s.stopped.Load() {
s.mu.Unlock()
return // Nobody would close this listener anymore
}
os.Remove(s.config.ListenUnix)
s.unixListener, err = net.Listen("unix", s.config.ListenUnix)
if err != nil {
@@ -420,6 +508,14 @@ func (s *Server) Run() error {
} else if s.config.EnableMetrics {
s.metricsHandler = promhttp.Handler()
}
if s.config.ClusterListen != "" {
// The cluster serves the whole private listener: every path on it is the peer API, which
// the cluster owns and authenticates itself
s.httpClusterServer = &http.Server{Addr: s.config.ClusterListen, Handler: s.cluster}
go func() {
errChan <- s.httpClusterServer.ListenAndServe()
}()
}
if s.config.ProfileListenHTTP != "" {
profileMux := http.NewServeMux()
profileMux.HandleFunc("/debug/pprof/", pprof.Index)
@@ -446,12 +542,33 @@ func (s *Server) Run() error {
return <-errChan
}
// Stop stops HTTP (+HTTPS) server and all managers
// Stop stops the HTTP (+HTTPS) server and all managers. It is idempotent: a signal handler and
// the serve command both call it, and the second call waits for the first to finish.
func (s *Server) Stop() {
s.stopped.Store(true) // Before the lock: Run must see this even if stop() is stuck
s.stopOnce.Do(s.stopBounded)
}
// stopBounded runs the shutdown and gives up on it after stopTimeout, so that a store which
// never finishes closing cannot keep the process alive
func (s *Server) stopBounded() {
done := make(chan struct{})
go func() {
s.stop()
close(done)
}()
select {
case <-done:
case <-time.After(stopTimeout):
log.Tag(tagStartup).Warn("Shutdown did not finish within %v, exiting anyway", stopTimeout)
}
}
func (s *Server) stop() {
s.mu.Lock()
defer s.mu.Unlock()
if s.httpServer != nil {
s.httpServer.Close()
for _, httpServer := range s.httpServers {
httpServer.Close()
}
if s.httpsServer != nil {
s.httpsServer.Close()
@@ -465,6 +582,12 @@ func (s *Server) Stop() {
if s.attachment != nil {
s.attachment.Close()
}
if s.httpClusterServer != nil {
s.httpClusterServer.Close()
}
if s.tail != nil {
s.tail.Close() // Before the databases close under it
}
s.closeDatabases()
if s.ban != nil {
s.ban.Close()
@@ -475,12 +598,14 @@ func (s *Server) Stop() {
}
func (s *Server) closeDatabases() {
if s.userManager != nil {
s.userManager.Close()
}
// Message cache first: it may still be writing its last batch, and on Postgres all stores
// share one pool, so closing any other store first would close the pool under that write
if s.messageCache != nil {
s.messageCache.Close()
}
if s.userManager != nil {
s.userManager.Close()
}
if s.webPush != nil {
s.webPush.Close()
}
@@ -520,7 +645,8 @@ func (s *Server) handleError(w http.ResponseWriter, r *http.Request, v *visitor,
}
metrics.HTTPRequests.WithLabelValues(strconv.Itoa(httpErr.HTTPCode), strconv.Itoa(httpErr.Code), r.Method).Inc()
isRateLimiting := util.Contains(rateLimitingErrorCodes, httpErr.HTTPCode)
isNormalError := strings.Contains(err.Error(), "i/o timeout") || util.Contains(normalErrorCodes, httpErr.HTTPCode)
isNormalError := strings.Contains(err.Error(), "i/o timeout") || util.Contains(normalErrorCodes, httpErr.HTTPCode) ||
httpErr.Code == errHTTPServiceUnavailableDraining.Code // A draining node rejects every new subscription, by design
ev := logvr(v, r).Err(err)
if websocket.IsWebSocketUpgrade(r) {
ev.Tag(tagWebsocket).Fields(websocketErrorContext(err))
@@ -567,6 +693,8 @@ func (s *Server) handleInternal(w http.ResponseWriter, r *http.Request, v *visit
return s.ensureWebEnabled(s.handleEmpty)(w, r, v)
} else if r.Method == http.MethodGet && r.URL.Path == apiHealthPath {
return s.handleHealth(w, r, v)
} else if r.Method == http.MethodGet && r.URL.Path == apiReadyPath {
return s.handleReady(w, r, v)
} else if r.Method == http.MethodGet && r.URL.Path == apiVersionPath {
return s.ensureAdmin(s.handleVersion)(w, r, v)
} else if r.Method == http.MethodGet && r.URL.Path == apiConfigPath {
@@ -587,6 +715,22 @@ func (s *Server) handleInternal(w http.ResponseWriter, r *http.Request, v *visit
return s.ensureAdmin(s.handleAccessAllow)(w, r, v)
} else if r.Method == http.MethodDelete && r.URL.Path == apiUsersAccessPath {
return s.ensureAdmin(s.handleAccessReset)(w, r, v)
} else if r.Method == http.MethodGet && r.URL.Path == apiUsersTokensPath {
return s.ensureAdmin(s.handleUsersTokensGet)(w, r, v)
} else if r.Method == http.MethodPost && r.URL.Path == apiUsersTokensPath {
return s.ensureAdmin(s.handleUsersTokensAdd)(w, r, v)
} else if r.Method == http.MethodDelete && r.URL.Path == apiUsersTokensPath {
return s.ensureAdmin(s.handleUsersTokensDelete)(w, r, v)
} else if r.Method == http.MethodPost && r.URL.Path == apiUsersPasswordResetPath {
return s.ensureAdmin(s.handleUsersPasswordReset)(w, r, v)
} else if r.Method == http.MethodGet && r.URL.Path == apiTiersPath && v.User().IsAdmin() {
return s.ensureAdmin(s.handleAdminTiersGet)(w, r, v) // Non-admins get the billing tiers, see below
} else if r.Method == http.MethodPost && r.URL.Path == apiTiersPath {
return s.ensureAdmin(s.handleAdminTiersAdd)(w, r, v)
} else if r.Method == http.MethodPut && r.URL.Path == apiTiersPath {
return s.ensureAdmin(s.handleAdminTiersUpdate)(w, r, v)
} else if r.Method == http.MethodDelete && r.URL.Path == apiTiersPath {
return s.ensureAdmin(s.handleAdminTiersDelete)(w, r, v)
} else if r.Method == http.MethodPost && r.URL.Path == apiAccountPath {
return s.ensureUserManager(s.handleAccountCreate)(w, r, v)
} else if r.Method == http.MethodGet && r.URL.Path == apiAccountPath {
@@ -653,8 +797,10 @@ func (s *Server) handleInternal(w http.ResponseWriter, r *http.Request, v *visit
return s.ensureWebPushEnabled(s.limitRequests(s.handleWebPushDelete))(w, r, v)
} else if r.Method == http.MethodGet && r.URL.Path == apiStatsPath {
return s.handleStats(w, r, v)
} else if r.Method == http.MethodGet && r.URL.Path == apiTiersPath {
} else if r.Method == http.MethodGet && r.URL.Path == apiAccountBillingTiersPath {
return s.ensurePaymentsEnabled(s.handleBillingTiersGet)(w, r, v)
} else if r.Method == http.MethodGet && r.URL.Path == apiTiersPath {
return s.ensurePaymentsEnabled(s.handleBillingTiersGet)(w, r, v) // Non-admins: web apps before v2.36.0, remove after 2026-11-22
} else if r.Method == http.MethodGet && r.URL.Path == matrixPushPath {
return s.handleMatrixDiscovery(w)
} else if r.Method == http.MethodGet && r.URL.Path == metricsPath && s.metricsHandler != nil {
@@ -717,10 +863,27 @@ func (s *Server) handleTopicAuth(w http.ResponseWriter, _ *http.Request, _ *visi
}
func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request, _ *visitor) error {
response := &apiHealthResponse{
Healthy: true,
// Unhealthy = the registry heartbeat went stale and peers stopped forwarding to this node. For
// monitoring; load balancers check /v1/ready, which keeps the only active node in
healthy := s.cluster.Healthy()
if !healthy {
w.WriteHeader(http.StatusServiceUnavailable)
}
return s.writeJSON(w, response)
return s.writeJSON(w, &apiHealthResponse{Healthy: healthy, Role: s.cluster.Role()})
}
// handleReady answers whether a load balancer should send clients here (see cluster.Cluster's
// Ready). A passive node is healthy (see handleHealth) but never ready.
func (s *Server) handleReady(w http.ResponseWriter, _ *http.Request, _ *visitor) error {
ready := s.cluster.Ready()
if !ready {
w.WriteHeader(http.StatusServiceUnavailable)
}
return s.writeJSON(w, &apiReadyResponse{
Ready: ready,
Role: s.cluster.Role(),
Draining: s.cluster.Draining(),
})
}
// handleMetrics returns Prometheus metrics. This endpoint is only called if enable-metrics is set,
@@ -824,6 +987,11 @@ func (s *Server) handleMatrixDiscovery(w http.ResponseWriter) error {
// the single choke point through which every published message must pass; t may be nil when
// the topic has no local subscribers (delayed sender).
func (s *Server) dispatch(v *visitor, t *topic, m *model.Message, opts dispatchOpts) error {
// Hand the message to the other cluster nodes: fire-and-forget, so a slow or dead peer never
// delays the publisher. Nop single-node.
if err := s.cluster.ForwardMessage(m); err != nil {
logvm(v, m).Err(err).Warn("Unable to forward message to cluster peers")
}
// Deliver to local subscribers
if t != nil {
if opts.async {
@@ -856,6 +1024,9 @@ func (s *Server) dispatch(v *visitor, t *topic, m *model.Message, opts dispatchO
}
func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*model.Message, error) {
if s.readOnly.Load() {
return nil, errHTTPServiceUnavailableDraining // Read-only; the load balancers stopped sending requests in the drain delay
}
start := time.Now()
t, err := fromContext[*topic](r, contextTopic)
if err != nil {
@@ -947,15 +1118,18 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*model.Mess
logvrm(v, r, m).Tag(tagPublish).Debug("Message delayed, will process later")
}
if cache {
// Delete any existing scheduled message with the same sequence ID
deletedIDs, err := s.messageCache.DeleteScheduledBySequenceID(t.ID, m.SequenceID)
if err != nil {
return nil, err
}
// Delete attachment files for deleted scheduled messages
if s.attachment != nil && len(deletedIDs) > 0 {
if err := s.attachment.Remove(deletedIDs...); err != nil {
logvrm(v, r, m).Tag(tagPublish).Err(err).Warn("Error removing attachments for deleted scheduled messages")
// Delete any existing scheduled message with the same sequence ID. Without a client-provided
// sequence ID, it is the message's own fresh ID and nothing can match, so skip the round trip.
if m.SequenceID != m.ID {
deletedIDs, err := s.messageCache.DeleteScheduledBySequenceID(t.ID, m.SequenceID)
if err != nil {
return nil, err
}
// Delete attachment files for deleted scheduled messages
if s.attachment != nil && len(deletedIDs) > 0 {
if err := s.attachment.Remove(deletedIDs...); err != nil {
logvrm(v, r, m).Tag(tagPublish).Err(err).Warn("Error removing attachments for deleted scheduled messages")
}
}
}
logvrm(v, r, m).Tag(tagPublish).Debug("Adding message to cache")
@@ -963,10 +1137,6 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*model.Mess
return nil, err
}
}
u := v.User()
if s.userManager != nil && u != nil && u.Tier != nil {
go s.userManager.EnqueueUserStats(u.ID, v.Stats())
}
s.mu.Lock()
s.messages++
s.mu.Unlock()
@@ -1235,7 +1405,15 @@ func (s *Server) parsePublishParams(r *http.Request, m *model.Message) (cache bo
if call != "" {
return false, false, "", "", "", false, "", errHTTPBadRequestDelayNoCall // we cannot store the phone number (yet)
}
delay, err := util.ParseFutureTime(delayStr, time.Now())
now := time.Now()
if timezone := readParam(r, "x-timezone", "timezone"); timezone != "" {
location, err := time.LoadLocation(timezone)
if err != nil {
return false, false, "", "", "", false, "", errHTTPBadRequestTimezoneInvalid
}
now = now.In(location)
}
delay, err := util.ParseFutureTime(delayStr, now)
if err != nil {
return false, false, "", "", "", false, "", errHTTPBadRequestDelayCannotParse
} else if delay.Unix() < time.Now().Add(s.config.MessageDelayMin).Unix() {
@@ -1253,7 +1431,7 @@ func (s *Server) parsePublishParams(r *http.Request, m *model.Message) (cache bo
}
}
contentType, markdown := readParam(r, "content-type", "content_type"), readBoolParam(r, false, "x-markdown", "markdown", "md")
if markdown || strings.ToLower(contentType) == "text/markdown" {
if markdown || isMarkdownContentType(contentType) {
m.ContentType = "text/markdown"
}
unifiedpush = readBoolParam(r, false, "x-unifiedpush", "unifiedpush", "up") // see GET too!
@@ -1432,6 +1610,8 @@ func (s *Server) handleSubscribeHTTP(w http.ResponseWriter, r *http.Request, v *
poll, since, scheduled, filters, err := parseSubscribeParams(r)
if err != nil {
return err
} else if !poll && s.readOnly.Load() {
return errHTTPServiceUnavailableDraining
}
var wlock sync.Mutex
var closed bool
@@ -1488,17 +1668,26 @@ func (s *Server) handleSubscribeHTTP(w http.ResponseWriter, r *http.Request, v *
meterPollBandwidth = true
return s.sendOldMessages(w, topics, since, scheduled, v, sub)
}
w.Header().Set("Incremental", "?1") // Ask proxies to forward the stream without buffering (RFC 10036)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if testHookBeforeSubscribe != nil {
testHookBeforeSubscribe()
}
streamID := s.streams.Add(cancel)
defer s.streams.Remove(streamID)
subscriberIDs := make([]int, 0)
for _, t := range topics {
subscriberIDs = append(subscriberIDs, t.Subscribe(sub, v.MaybeUserID(), cancel))
subscriberIDs = append(subscriberIDs, t.Subscribe(sub, v.User(), cancel))
}
defer func() {
for i, subscriberID := range subscriberIDs {
topics[i].Unsubscribe(subscriberID) // Order!
}
}()
if err := s.authorizeSubscribed(v, topics); err != nil {
return err
}
if err := sub(v, model.NewOpenMessage(topicsStr)); err != nil { // Send out open message
return err
}
@@ -1546,6 +1735,8 @@ func (s *Server) handleSubscribeWS(w http.ResponseWriter, r *http.Request, v *vi
poll, since, scheduled, filters, err := parseSubscribeParams(r)
if err != nil {
return err
} else if !poll && s.readOnly.Load() {
return errHTTPServiceUnavailableDraining // Before the upgrade, so the load balancer can retry elsewhere
}
upgrader := &websocket.Upgrader{
ReadBufferSize: wsBufferSize,
@@ -1639,15 +1830,23 @@ func (s *Server) handleSubscribeWS(w http.ResponseWriter, r *http.Request, v *vi
}
return s.sendOldMessages(w, topics, since, scheduled, v, sub)
}
if testHookBeforeSubscribe != nil {
testHookBeforeSubscribe()
}
streamID := s.streams.Add(cancel)
defer s.streams.Remove(streamID)
subscriberIDs := make([]int, 0)
for _, t := range topics {
subscriberIDs = append(subscriberIDs, t.Subscribe(sub, v.MaybeUserID(), cancel))
subscriberIDs = append(subscriberIDs, t.Subscribe(sub, v.User(), cancel))
}
defer func() {
for i, subscriberID := range subscriberIDs {
topics[i].Unsubscribe(subscriberID) // Order!
}
}()
if err := s.authorizeSubscribed(v, topics); err != nil {
return err
}
if err := sub(v, model.NewOpenMessage(topicsStr)); err != nil { // Send out open message
return err
}
@@ -1750,10 +1949,24 @@ func (s *Server) sendOldMessages(w http.ResponseWriter, topics []*topic, since m
if since.IsNone() {
return nil
}
var marker *message.Marker
if since.IsID() {
var err error
if marker, err = s.messageCache.ResolveMarker(since.ID()); err != nil {
return err
}
}
messages := make([]*model.Message, 0)
truncated := false
for _, t := range topics {
topicMessages, topicTruncated, err := s.messageCache.MessagesCapped(t.ID, since, scheduled, s.config.MessagePollSizeLimit)
var topicMessages []*model.Message
var topicTruncated bool
var err error
if marker != nil {
topicMessages, topicTruncated, err = s.messageCache.MessagesAfterMarker(t.ID, marker, scheduled, s.config.MessagePollSizeLimit)
} else {
topicMessages, topicTruncated, err = s.messageCache.MessagesCapped(t.ID, sinceForTopic(t, since), scheduled, s.config.MessagePollSizeLimit)
}
if err != nil {
return err
}
@@ -1834,7 +2047,11 @@ func (s *Server) topicsFromPath(v *visitor, path string) ([]*topic, string, erro
if len(parts) < 2 {
return nil, "", errHTTPBadRequestTopicInvalid
}
topicIDs := util.SplitNoEmpty(parts[1], ",")
// De-duplicate, or each repetition of a topic replays (and buffers) its cache again
topicIDs := util.Dedup(util.SplitNoEmpty(parts[1], ","))
if len(topicIDs) > subscribeTopicsLimit {
return nil, "", errHTTPBadRequestTooManyTopics
}
topics, err := s.topicsFromIDs(v, topicIDs...)
if err != nil {
return nil, "", err
@@ -1888,14 +2105,12 @@ func (s *Server) topicFromID(v *visitor, id string) (*topic, error) {
// topicsFromPattern returns a list of topics matching the given pattern, but it does not create them.
func (s *Server) topicsFromPattern(pattern string) ([]*topic, error) {
s.mu.RLock()
defer s.mu.RUnlock()
patternRegexp, err := regexp.Compile("^" + strings.ReplaceAll(pattern, "*", ".*") + "$")
if err != nil {
return nil, err
}
topics := make([]*topic, 0)
for _, t := range s.topics {
for _, t := range s.allTopics() {
if patternRegexp.MatchString(t.ID) {
topics = append(topics, t)
}
@@ -1903,6 +2118,18 @@ func (s *Server) topicsFromPattern(pattern string) ([]*topic, error) {
return topics, nil
}
// allTopics returns all topics. Callers that scan them (~313k on ntfy.sh) do so without s.mu held,
// which every request that creates a visitor or topic needs.
func (s *Server) allTopics() []*topic {
s.mu.RLock()
defer s.mu.RUnlock()
topics := make([]*topic, 0, len(s.topics))
for _, t := range s.topics {
topics = append(topics, t)
}
return topics
}
func (s *Server) runSMTPServer() error {
s.smtpServerBackend = newMailBackend(s.config, s.handle)
s.smtpServer = smtp.NewServer(s.smtpServerBackend)
@@ -1953,9 +2180,7 @@ func (s *Server) resetStats() {
log.Info("Resetting all visitor stats (daily task)")
s.mu.Lock()
defer s.mu.Unlock() // Includes the database query to avoid races with other processes
for _, v := range s.visitors {
v.ResetStats()
}
s.limiter.ResetUsage()
if s.userManager != nil {
if err := s.userManager.ResetStats(); err != nil {
log.Tag(tagResetter).Warn("Failed to write to database: %s", err.Error())
@@ -1967,10 +2192,15 @@ func (s *Server) runFirebaseKeepaliver() {
if s.firebaseClient == nil {
return
}
v := newVisitor(s.config, s.messageCache, s.userManager, netip.IPv4Unspecified(), nil) // Background process, not a real visitor, uses IP 0.0.0.0
v := newVisitor(s.visitors, netip.IPv4Unspecified(), nil) // Background process, not a real visitor, uses IP 0.0.0.0
for {
select {
case <-time.After(s.config.FirebaseKeepaliveInterval):
// Leader only: a keepalive wakes every subscribed phone, so the cluster sends it
// once rather than once per node. Per tick, to survive a failover.
if !s.cluster.IsLeader() {
continue
}
s.sendToFirebase(v, model.NewKeepaliveMessage(firebaseControlTopic))
/*
FIXME: Disable iOS polling entirely for now due to thundering herd problem (see #677)
@@ -1990,6 +2220,11 @@ func (s *Server) runDelayedSender() {
for {
select {
case <-time.After(s.config.DelayedSenderInterval):
// Leader only, to keep delayed sends on one node. A preference, not what makes
// delivery safe: MessagesDue claims each row, so a non-leader is harmless.
if !s.cluster.IsLeader() {
continue
}
if err := s.sendDelayedMessages(); err != nil {
log.Tag(tagPublish).Err(err).Warn("Error sending delayed messages")
}
@@ -2000,6 +2235,9 @@ func (s *Server) runDelayedSender() {
}
func (s *Server) sendDelayedMessages() error {
if s.cluster.Draining() {
return nil // On its way out: claiming now could leave a claimed message unsent; the next leader sends it
}
messages, err := s.messageCache.MessagesDue()
if err != nil {
return err
@@ -2096,6 +2334,9 @@ func (s *Server) transformBodyJSON(next handleFunc) handleFunc {
if m.Delay != "" {
r.Header.Set("X-Delay", m.Delay)
}
if m.Timezone != "" {
r.Header.Set("X-Timezone", m.Timezone)
}
if m.Call != "" {
r.Header.Set("X-Call", m.Call)
}
@@ -2130,18 +2371,10 @@ func (s *Server) transformMatrixJSON(next handleFunc) handleFunc {
}
}
// visitor returns the visitor for this request. It is built per request, so concurrent requests from
// the same IP (which share a limiter key) cannot see each other's user (#1051).
func (s *Server) visitor(ip netip.Addr, user *user.User) *visitor {
s.mu.Lock()
defer s.mu.Unlock()
id := visitorID(ip, user, s.config)
v, exists := s.visitors[id]
if !exists {
s.visitors[id] = newVisitor(s.config, s.messageCache, s.userManager, ip, user)
return s.visitors[id]
}
v.Keepalive()
v.SetUser(user) // Always update with the latest user, may be nil!
return v
return newVisitor(s.visitors, ip, user)
}
func (s *Server) writeJSON(w http.ResponseWriter, v any) error {
@@ -2160,8 +2393,23 @@ func (s *Server) updateAndWriteStats(messagesCount int64) {
if len(s.messagesHistory) > messagesHistoryMax {
s.messagesHistory = s.messagesHistory[1:]
}
snapshot, delta := s.messages, s.messages-s.messagesFlushed
s.mu.Unlock()
if err := s.messageCache.UpdateStats(messagesCount); err != nil {
// Write this node's delta and fold the cluster-wide total back in, so every node's counter
// (and /v1/stats) converges on the shared sum instead of overwriting it
if err := s.messageCache.AddStats(delta); err != nil {
log.Tag(tagManager).Err(err).Warn("Cannot write messages stats")
return // Delta remains unflushed; retried on the next tick
}
total, err := s.messageCache.Stats()
if err != nil {
log.Tag(tagManager).Err(err).Warn("Cannot read messages stats")
total = snapshot // Keep the local view; only the flush marker moves
}
s.mu.Lock()
// The marker is the folded total (everything the database already has, incl. peer counts),
// so the next delta is only this node's new publishes; peer counts must never be re-added
s.messagesFlushed = total
s.messages = total + (s.messages - snapshot) // Publishes that arrived while flushing stay counted
s.mu.Unlock()
}
server/server.yml
+53 -3
View File
@@ -19,6 +19,9 @@
# To listen on all interfaces, you may omit the IP address, e.g. ":443".
# To disable HTTP, set "listen-http" to "-".
#
# "listen-http" may list several comma-separated addresses, e.g. "10.0.0.5:2586,10.0.0.5:2588".
# Behind a load balancer, each address allows ~63k more connections from it (one per source port).
#
# listen-http: ":80"
# listen-https:
@@ -61,6 +64,53 @@
#
# database-url: <connection-string>
# EXPERIMENTAL: Clustering options. Setting "experimental-cluster-listen" runs this server as one
# node of a cluster that shares a PostgreSQL database. This is not ready for production: dropped
# node-to-node batches are not recovered yet, and rate limits and quotas are enforced per node,
# not across the cluster. The "experimental-" prefix means these option names may still change.
#
# - experimental-cluster-listen is the ip:port of the dedicated fan-out listener that peer nodes
# talk to. Bind it to a private network interface (e.g. "10.0.0.5:2587"); the public listeners
# never serve the fan-out endpoint. Setting it is what enables clustering.
# - experimental-cluster-node-id is a stable per-node identifier (e.g. the hostname); required.
# - experimental-cluster-advertise-url is the base URL peer nodes use to reach this node's
# fan-out listener; it defaults to "http://<experimental-cluster-listen>". It must be set
# explicitly if the listen address binds a wildcard address (e.g. ":2587").
# - experimental-cluster-secret authenticates node-to-node fan-out requests; it must be
# identical on all nodes.
# - experimental-cluster-batch-linger is how long fan-out messages may wait to form a batch per
# peer node, trading up to that much cross-node delivery latency for a bounded request rate
# between nodes. Set to 0 to send each message immediately.
# - experimental-cluster-role is "active" (default) or "passive". A passive node is meant to run
# outside the load balancer (e.g. to observe a new node under real load): it receives fan-out
# and serves clients that connect to it directly, but it never leads the cluster's singleton
# jobs, and it never closes its subscribers to send them elsewhere. Active nodes do not count
# it as a place to send theirs either. The member list reports each node's role, so load
# balancer agents can route to active nodes only.
# - experimental-cluster-drain-delay and experimental-cluster-drain-duration control how an active
# node drains on shutdown (SIGTERM) if another active node is up, so its clients move to the
# other nodes gradually instead of all at once. For the drain delay, the node withdraws: it is not
# ready (/v1/ready fails) but still serves everything, so the load balancer can stop sending it
# requests. Then it is read-only (new subscriptions and publishes get 503) and closes its
# subscriber connections evenly over the drain duration (all at once with 0). Make sure the
# service manager waits long enough (systemd: TimeoutStopSec, default 90s).
#
# Clustering also requires "database-url": the shared PostgreSQL database carries the durable
# state and the cluster's own coordination tables.
#
# SECURITY: The fan-out endpoint injects messages into arbitrary topics. The shared secret
# protects it, and it is only served on the dedicated cluster listener -- but you should still
# make sure that listener is reachable only from the private network (firewall/VPC rules).
#
# experimental-cluster-listen: <ip:port>
# experimental-cluster-node-id: <hostname>
# experimental-cluster-advertise-url: "http://<ip:port>"
# experimental-cluster-secret: <secret>
# experimental-cluster-batch-linger: 500ms
# experimental-cluster-role: active
# experimental-cluster-drain-delay: 15s
# experimental-cluster-drain-duration: 1m
# If "cache-file" is set, messages are cached in a local SQLite database instead of only in-memory.
# This allows for service restarts without losing messages in support of the since= parameter.
# Not required if "database-url" is set (messages are stored in PostgreSQL instead).
@@ -90,7 +140,7 @@
#
# Check your permissions:
# If you are running ntfy with systemd, make sure this cache file is owned by the
# ntfy user and group by running: chown ntfy.ntfy <filename>.
# ntfy user and group by running: chown ntfy:ntfy <filename>.
#
# cache-file: <filename>
# cache-duration: "12h"
@@ -125,7 +175,7 @@
#
# Check your permissions:
# If you are running ntfy with systemd, make sure this user database file is owned by the
# ntfy user and group by running: chown ntfy.ntfy <filename>.
# ntfy user and group by running: chown ntfy:ntfy <filename>.
#
# auth-file: <filename>
# auth-default-access: "read-write"
@@ -494,7 +544,7 @@
#
# Check your permissions:
# If you are running ntfy with systemd, make sure this log file is owned by the
# ntfy user and group by running: chown ntfy.ntfy <filename>.
# ntfy user and group by running: chown ntfy:ntfy <filename>.
#
# Example (good for production):
# log-level: info
server/server_account.go
+9 -1
View File
@@ -245,6 +245,7 @@ func (s *Server) handleAccountDelete(w http.ResponseWriter, r *http.Request, v *
if err := s.userManager.MarkUserRemoved(u); err != nil {
return err
}
s.userDeleted(u)
return s.writeJSON(w, newSuccessResponse())
}
@@ -324,6 +325,11 @@ func (s *Server) handleAccountTokenCreate(w http.ResponseWriter, r *http.Request
func (s *Server) handleAccountTokenUpdate(w http.ResponseWriter, r *http.Request, v *visitor) error {
u := v.User()
if u == nil {
// ensureUser already checked; kept as a guard, since a nil user here was a recurring
// production panic while visitors were shared between requests (#1051)
return errHTTPUnauthorized
}
req, err := readJSONWithLimit[apiAccountTokenUpdateRequest](r.Body, jsonBodyBytesLimit, true) // Allow empty body!
if err != nil {
return err
@@ -547,7 +553,7 @@ func (s *Server) handleAccountReservationAdd(w http.ResponseWriter, r *http.Requ
if err != nil {
return err
}
t.CancelSubscribersExceptUser(u.ID)
t.CancelSubscribersExceptUser(u.ID) // The other nodes end theirs when they reload their access cache, see broadcastAccessChanged
return s.writeJSON(w, newSuccessResponse())
}
@@ -650,6 +656,8 @@ func (s *Server) handleAccountPhoneNumberAdd(w http.ResponseWriter, r *http.Requ
if err := s.twilio.CheckVerify(req.Number, req.Code); err != nil {
if errors.Is(err, twilio.ErrVerificationExpired) {
return errHTTPGonePhoneVerificationExpired
} else if errors.Is(err, twilio.ErrVerificationCodeInvalid) {
return errHTTPBadRequestPhoneVerificationCodeInvalid
}
return err
}
server/server_account_test.go
+215
View File
@@ -3,11 +3,15 @@ package server
import (
"fmt"
"github.com/stretchr/testify/require"
dbtest "heckel.io/ntfy/v2/db/test"
"heckel.io/ntfy/v2/log"
"heckel.io/ntfy/v2/model"
"heckel.io/ntfy/v2/user"
"heckel.io/ntfy/v2/util"
"io"
"net"
"net/http"
"net/http/httptest"
"net/netip"
"os"
"path/filepath"
@@ -522,6 +526,31 @@ func TestAccount_Delete_Success(t *testing.T) {
})
}
func TestAccount_Delete_EndsStreamsEverywhere(t *testing.T) {
// Deleting your own account ends your streams here and tells the other cluster nodes, like an
// admin's delete: an admin without access entries leaves nothing for an access reload to find
conf := newTestConfigWithAuthFile(t, "")
s := newTestServer(t, conf)
cl := &fakeCluster{}
s.cluster = cl
require.Nil(t, s.userManager.AddUser("phil", "mypass", user.RoleAdmin, false))
phil, err := s.userManager.User("phil")
require.Nil(t, err)
headers := map[string]string{
"Authorization": util.BasicAuth("phil", "mypass"),
}
stream := subscribeStream(t, s, "mytopic", headers)
rr := request(t, s, "DELETE", "/v1/account", `{"password":"mypass"}`, headers)
require.Equal(t, 200, rr.Code)
select {
case <-stream.ended:
case <-time.After(3 * time.Second):
t.Fatal("the deleted user's stream stayed open")
}
require.Equal(t, []string{phil.ID}, cl.DeletedUsers())
}
func TestAccount_Delete_Not_Allowed(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
conf := newTestConfigWithAuthFile(t, databaseURL)
@@ -898,3 +927,189 @@ func TestAccount_Reservation_Delete_Messages_And_Attachments(t *testing.T) {
account, _ = util.UnmarshalJSON[apiAccountResponse](io.NopCloser(rr.Body))
require.Equal(t, int64(2), account.Stats.Messages) // Is not reset!
}*/
func TestAccount_TokenUpdate_NilUser_NoPanic(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
// Regression for a recurring production panic in handleAccountTokenUpdate: the handler
// re-reads v.User() after ensureUser checked it, and must return 401 for a nil user, not
// dereference it. Visitors were shared between requests then (#1051).
conf := newTestConfigWithAuthFile(t, databaseURL)
conf.EnableLogin = true
s := newTestServer(t, conf)
ip := netip.MustParseAddr("9.9.9.9")
require.Nil(t, s.userManager.AddUser("phil", "phil12345", user.RoleUser, false))
u, err := s.userManager.User("phil")
require.Nil(t, err)
authed := s.visitor(ip, u)
anon := s.visitor(ip, nil) // A concurrent anonymous request from the same IP
require.NotNil(t, authed.User())
r, _ := http.NewRequest("PATCH", "/v1/account/token", strings.NewReader("{}"))
rr := httptest.NewRecorder()
err = s.handleAccountTokenUpdate(rr, r, anon)
require.Error(t, err)
httpErr, ok := err.(*errHTTP)
require.True(t, ok)
require.Equal(t, 401, httpErr.HTTPCode)
})
}
func TestAccount_Reservation_TakeoverCancelsPeerSubscribers(t *testing.T) {
// Reserving a public topic on node A must end an anonymous stream on node B, and B must deny
// the reconnect right away: its access cache would otherwise still say "public" until the
// next periodic reload
sA, sB := newReservationTestNodes(t, true, time.Hour)
stream := subscribeStream(t, sB, "mytopic", nil)
reserveDenyAll(t, sA, "mytopic")
select {
case <-stream.ended:
case <-time.After(5 * time.Second):
t.Fatal("anonymous stream on node B was not canceled after the reservation on node A")
}
require.Equal(t, 403, request(t, sB, "GET", "/mytopic/json?poll=1", "", nil).Code)
// The owner's messages, published on either node, stay private
phil := map[string]string{"Authorization": util.BasicAuth("phil", "phil")}
require.Equal(t, 200, request(t, sA, "PUT", "/mytopic", "secret from A", phil).Code)
require.Equal(t, 200, request(t, sB, "PUT", "/mytopic", "secret from B", phil).Code)
require.NotContains(t, stream.body(), "secret")
}
func TestAccount_Reservation_LostCancelCaughtByAccessReload(t *testing.T) {
// The relay is best effort: if node B never hears about the reservation (here: the nodes are
// not even clustered), B's next access cache reload finds the changed entries and ends the
// streams that may no longer read the topic
sA, sB := newReservationTestNodes(t, false, 500*time.Millisecond)
stream := subscribeStream(t, sB, "mytopic", nil)
reserveDenyAll(t, sA, "mytopic")
select {
case <-stream.ended:
case <-time.After(5 * time.Second):
t.Fatal("anonymous stream on node B was not canceled by its access cache reload")
}
require.Equal(t, 403, request(t, sB, "GET", "/mytopic/json?poll=1", "", nil).Code)
}
// newReservationTestNodes starts nodes A and B on one Postgres schema, with reservations and the
// access cache on as on ntfy.sh, and user phil (tier with reservations) to reserve topics.
// With clustered, A tells B about its access changes; without, B never hears about them.
func newReservationTestNodes(t *testing.T, clustered bool, accessReloadInterval time.Duration) (sA, sB *Server) {
schemaDSN := dbtest.CreateTestPostgresSchema(t)
newConf := func() *Config {
conf := newTestConfig(t, schemaDSN)
conf.AuthDefault = user.PermissionReadWrite
conf.AuthAccessCacheEnabled = true
conf.AuthAccessCacheReloadInterval = accessReloadInterval
conf.EnableReservations = true
return conf
}
confB := newConf()
if clustered {
listenerB, err := net.Listen("tcp", "127.0.0.1:0")
require.Nil(t, err)
confB.ClusterNodeID = "node-b"
confB.ClusterListen = listenerB.Addr().String()
confB.ClusterSecret = "s3cret"
confB.ClusterAdvertiseURL = "http://" + listenerB.Addr().String()
sB = newTestServer(t, confB)
srvB := &http.Server{Handler: sB.cluster}
go srvB.Serve(listenerB)
t.Cleanup(func() {
srvB.Close()
})
} else {
sB = newTestServer(t, confB)
}
confA := newConf()
if clustered {
confA.ClusterNodeID = "node-a"
confA.ClusterListen = "127.0.0.1:1" // Enables clustering; A only sends in these tests
confA.ClusterSecret = "s3cret"
confA.ClusterAdvertiseURL = "http://127.0.0.1:1"
}
sA = newTestServer(t, confA)
require.Nil(t, sA.userManager.AddTier(&user.Tier{
Code: "pro",
MessageLimit: 100,
ReservationLimit: 2,
}))
require.Nil(t, sA.userManager.AddUser("phil", "phil", user.RoleUser, false))
require.Nil(t, sA.userManager.ChangeTier("phil", "pro"))
if clustered {
waitFor(t, func() bool {
return len(sA.cluster.Members()) == 2
})
}
return sA, sB
}
func reserveDenyAll(t *testing.T, s *Server, topic string) {
response := request(t, s, "POST", "/v1/account/reservation", `{"topic":"`+topic+`","everyone":"deny-all"}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 200, response.Code)
}
// TestAccount_PhoneNumberAdd_WrongCode ensures that a wrong verification code is rejected, and
// that the phone number is not added. Twilio answers a wrong code with 200 OK and status "pending".
func TestAccount_PhoneNumberAdd_WrongCode(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
twilioVerifyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
require.Nil(t, err)
switch r.URL.Path {
case "/v2/Services/VA1234567890/Verifications":
w.WriteHeader(http.StatusCreated)
w.Write([]byte(`{"status":"pending","valid":false}`))
case "/v2/Services/VA1234567890/VerificationCheck":
if string(body) == "Code=123456&To=%2B12223334444" {
w.Write([]byte(`{"status":"approved","valid":true}`))
} else {
w.Write([]byte(`{"status":"pending","valid":false}`))
}
default:
t.Fatal("Unexpected path:", r.URL.Path)
}
}))
defer twilioVerifyServer.Close()
c := newTestConfigWithAuthFile(t, databaseURL)
c.TwilioVerifyBaseURL = twilioVerifyServer.URL
c.TwilioCallsBaseURL = "http://127.0.0.1:1" // Never called
c.TwilioAccount = "AC1234567890"
c.TwilioAuthToken = "AAEAA1234567890"
c.TwilioPhoneNumber = "+1234567890"
c.TwilioVerifyService = "VA1234567890"
s := newTestServer(t, c)
require.Nil(t, s.userManager.AddTier(&user.Tier{Code: "pro", MessageLimit: 10, CallLimit: 10}))
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleUser, false))
require.Nil(t, s.userManager.ChangeTier("phil", "pro"))
u, err := s.userManager.User("phil")
require.Nil(t, err)
// Send verification code, then submit a wrong code
rr := request(t, s, "PUT", "/v1/account/phone/verify", `{"number":"+12223334444","channel":"sms"}`, map[string]string{
"authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 200, rr.Code)
rr = request(t, s, "PUT", "/v1/account/phone", `{"number":"+12223334444","code":"000000"}`, map[string]string{
"authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 400, rr.Code)
require.Equal(t, 40060, toHTTPError(t, rr.Body.String()).Code)
phoneNumbers, err := s.userManager.PhoneNumbers(u.ID)
require.Nil(t, err)
require.Empty(t, phoneNumbers)
// The correct code still works
rr = request(t, s, "PUT", "/v1/account/phone", `{"number":"+12223334444","code":"123456"}`, map[string]string{
"authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 200, rr.Code)
phoneNumbers, err = s.userManager.PhoneNumbers(u.ID)
require.Nil(t, err)
require.Equal(t, []string{"+12223334444"}, phoneNumbers)
})
}
server/server_admin.go
+511 -55
View File
@@ -2,8 +2,20 @@ package server
import (
"errors"
"heckel.io/ntfy/v2/user"
"fmt"
"net/http"
"net/netip"
"slices"
"strings"
"time"
"heckel.io/ntfy/v2/cluster"
"heckel.io/ntfy/v2/log"
"heckel.io/ntfy/v2/user"
)
const (
apiTierReset = "-" // Tier value that removes a user's tier (PUT /v1/users); same marker the CLI uses
)
func (s *Server) handleVersion(w http.ResponseWriter, r *http.Request, v *visitor) error {
@@ -25,22 +37,26 @@ func (s *Server) handleUsersGet(w http.ResponseWriter, r *http.Request, v *visit
}
usersResponse := make([]*apiUserResponse, len(users))
for i, u := range users {
tier := ""
if u.Tier != nil {
tier = u.Tier.Code
}
userGrants := make([]*apiUserGrantResponse, len(grants[u.ID]))
for i, g := range grants[u.ID] {
userGrants[i] = &apiUserGrantResponse{
Topic: g.TopicPattern,
Permission: g.Permission.String(),
Topic: g.TopicPattern,
Permission: g.Permission.String(),
Provisioned: g.Provisioned,
}
}
usersResponse[i] = &apiUserResponse{
Username: u.Name,
Role: string(u.Role),
Tier: tier,
Grants: userGrants,
Username: u.Name,
Role: string(u.Role),
Provisioned: u.Provisioned,
Grants: userGrants,
}
if u.Tier != nil {
usersResponse[i].Tier = u.Tier.Code
usersResponse[i].TierName = u.Tier.Name
}
if u.Name == user.Everyone {
usersResponse[i].DefaultAccess = s.userManager.DefaultAccess().String()
}
}
return s.writeJSON(w, usersResponse)
@@ -51,7 +67,13 @@ func (s *Server) handleUsersAdd(w http.ResponseWriter, r *http.Request, v *visit
if err != nil {
return err
} else if !user.AllowedUsername(req.Username) || (req.Password == "" && req.Hash == "") {
return errHTTPBadRequest.Wrap("username invalid, or password/password_hash missing")
return errHTTPBadRequest.Wrap("username invalid, or \"password\" or \"hash\" missing")
}
role := user.RoleUser
if req.Role != "" {
if role, err = checkRoleChange(r, req.Role); err != nil {
return err
}
}
u, err := s.userManager.User(req.Username)
if err != nil && !errors.Is(err, user.ErrUserNotFound) {
@@ -59,25 +81,22 @@ func (s *Server) handleUsersAdd(w http.ResponseWriter, r *http.Request, v *visit
} else if u != nil {
return errHTTPConflictUserExists
}
var tier *user.Tier
if req.Tier != "" {
tier, err = s.userManager.Tier(req.Tier)
if errors.Is(err, user.ErrTierNotFound) {
return errHTTPBadRequestTierInvalid
} else if err != nil {
return err
if _, err := s.userManager.Tier(req.Tier); err != nil {
return mapUserErrorToHTTP(err)
}
}
password, hashed := req.Password, false
if req.Hash != "" {
password, hashed = req.Hash, true
}
if err := s.userManager.AddUser(req.Username, password, user.RoleUser, hashed); err != nil {
return err
adminLog(v, r).Info("Adding user %s (role %s)", req.Username, role)
if err := s.userManager.AddUser(req.Username, password, role, hashed); err != nil {
return mapUserErrorToHTTP(err)
}
if tier != nil {
if req.Tier != "" {
if err := s.userManager.ChangeTier(req.Username, req.Tier); err != nil {
return err
return mapUserErrorToHTTP(err)
}
}
return s.writeJSON(w, newSuccessResponse())
@@ -89,42 +108,66 @@ func (s *Server) handleUsersUpdate(w http.ResponseWriter, r *http.Request, v *vi
return err
} else if !user.AllowedUsername(req.Username) {
return errHTTPBadRequest.Wrap("username invalid")
} else if req.Password == "" && req.Hash == "" && req.Tier == "" {
return errHTTPBadRequest.Wrap("need to provide at least one of \"password\", \"password_hash\" or \"tier\"")
} else if req.Password == "" && req.Hash == "" && req.Tier == "" && req.Role == "" {
return errHTTPBadRequest.Wrap("need to provide at least one of \"password\", \"hash\", \"tier\" or \"role\"")
}
var role user.Role
if req.Role != "" {
if role, err = checkRoleChange(r, req.Role); err != nil {
return err
}
}
if req.Tier != "" && req.Tier != apiTierReset {
if _, err := s.userManager.Tier(req.Tier); err != nil {
return mapUserErrorToHTTP(err)
}
}
u, err := s.userManager.User(req.Username)
if err != nil && !errors.Is(err, user.ErrUserNotFound) {
return err
} else if u != nil {
if u.IsAdmin() {
return errHTTPForbidden
if u.IsAdmin() && !isAdminSocketRequest(r) {
return errHTTPForbiddenAdminSocketOnly
}
if req.Hash != "" {
if err := s.userManager.ChangePassword(req.Username, req.Hash, true); err != nil {
return err
if req.Hash != "" || req.Password != "" {
adminLog(v, r).Info("Changing password of user %s", req.Username)
if req.Hash != "" {
err = s.userManager.ChangePassword(req.Username, req.Hash, true)
} else {
err = s.userManager.ChangePassword(req.Username, req.Password, false)
}
} else if req.Password != "" {
if err := s.userManager.ChangePassword(req.Username, req.Password, false); err != nil {
return err
if err != nil {
return mapUserErrorToHTTP(err)
}
}
} else if req.Role != "" {
return errHTTPBadRequestUserNotFound // Role changes only apply to existing users, see apiUserAddOrUpdateRequest
} else {
// Upsert: the admin UI creates users with PUT
password, hashed := req.Password, false
if req.Hash != "" {
password, hashed = req.Hash, true
}
adminLog(v, r).Info("Adding user %s (role %s)", req.Username, user.RoleUser)
if err := s.userManager.AddUser(req.Username, password, user.RoleUser, hashed); err != nil {
return err
return mapUserErrorToHTTP(err)
}
}
if req.Tier != "" {
if _, err = s.userManager.Tier(req.Tier); errors.Is(err, user.ErrTierNotFound) {
return errHTTPBadRequestTierInvalid
} else if err != nil {
return err
if req.Tier == apiTierReset {
adminLog(v, r).Info("Removing tier of user %s", req.Username)
if err := s.userManager.ResetTier(req.Username); err != nil {
return mapUserErrorToHTTP(err)
}
} else if req.Tier != "" {
adminLog(v, r).Info("Changing tier of user %s to %s", req.Username, req.Tier)
if err := s.userManager.ChangeTier(req.Username, req.Tier); err != nil {
return err
return mapUserErrorToHTTP(err)
}
}
if req.Role != "" {
adminLog(v, r).Info("Changing role of user %s to %s", req.Username, role)
if err := s.userManager.ChangeRole(req.Username, role); err != nil {
return mapUserErrorToHTTP(err)
}
}
return s.writeJSON(w, newSuccessResponse())
@@ -136,19 +179,23 @@ func (s *Server) handleUsersDelete(w http.ResponseWriter, r *http.Request, v *vi
return err
}
u, err := s.userManager.User(req.Username)
if errors.Is(err, user.ErrUserNotFound) {
return errHTTPBadRequestUserNotFound
} else if err != nil {
return err
} else if !u.IsUser() {
if err != nil {
return mapUserErrorToHTTP(err)
} else if !u.IsUser() && !isAdminSocketRequest(r) {
return errHTTPUnauthorized.Wrap("can only remove regular users from API")
} else if u.Name == user.Everyone {
return errHTTPBadRequest.Wrap("cannot remove the everyone user")
}
adminLog(v, r).Info("Removing user %s", req.Username)
if err := s.userManager.RemoveUser(req.Username); err != nil {
return err
return mapUserErrorToHTTP(err)
}
if err := s.killUserSubscriber(u, "*"); err != nil { // FIXME super inefficient
return err
if s.webPush != nil {
if err := s.webPush.RemoveSubscriptionsByUserID(u.ID); err != nil {
adminLog(v, r).Err(err).Warn("Failed to remove web push subscriptions of user %s", req.Username)
}
}
s.userDeleted(u) // FIXME super inefficient
return s.writeJSON(w, newSuccessResponse())
}
@@ -157,18 +204,19 @@ func (s *Server) handleAccessAllow(w http.ResponseWriter, r *http.Request, v *vi
if err != nil {
return err
}
_, err = s.userManager.User(req.Username)
if errors.Is(err, user.ErrUserNotFound) {
return errHTTPBadRequestUserNotFound
} else if err != nil {
return err
u, err := s.userManager.User(req.Username)
if err != nil {
return mapUserErrorToHTTP(err)
} else if u.IsAdmin() {
return errHTTPBadRequestAccessAdminUser
}
permission, err := user.ParsePermission(req.Permission)
if err != nil {
return errHTTPBadRequestPermissionInvalid
}
adminLog(v, r).Info("Granting %s access to topic %s for user %s", permission, req.Topic, req.Username)
if err := s.userManager.AllowAccess(req.Username, req.Topic, permission); err != nil {
return err
return mapUserErrorToHTTP(err)
}
return s.writeJSON(w, newSuccessResponse())
}
@@ -177,17 +225,241 @@ func (s *Server) handleAccessReset(w http.ResponseWriter, r *http.Request, v *vi
req, err := readJSONWithLimit[apiAccessResetRequest](r.Body, jsonBodyBytesLimit, false)
if err != nil {
return err
} else if req.Username == "" && req.Topic != "" {
return errHTTPBadRequest.Wrap("topic requires a username")
} else if req.Username == "" && !isAdminSocketRequest(r) {
return errHTTPForbiddenAdminSocketOnly // Resetting the whole ACL also opens all reserved topics
}
var u *user.User
if req.Username != "" {
if u, err = s.userManager.User(req.Username); err != nil {
return mapUserErrorToHTTP(err)
}
}
adminLog(v, r).Info("Resetting access for user %s, topic %s", req.Username, req.Topic)
if err := s.userManager.ResetAccess(req.Username, req.Topic); err != nil {
return mapUserErrorToHTTP(err)
}
// Without a username, the whole ACL is gone. Subscribers are not killed here; the periodic
// re-authorization of long-lived subscriptions catches up with them.
if u != nil {
topicPattern := req.Topic
if topicPattern == "" {
topicPattern = "*"
}
if err := s.killUserSubscriber(u, topicPattern); err != nil { // This may be a pattern
return err
}
}
return s.writeJSON(w, newSuccessResponse())
}
func (s *Server) handleUsersTokensGet(w http.ResponseWriter, r *http.Request, v *visitor) error {
var users []*user.User
if username := r.URL.Query().Get("username"); username != "" {
u, err := s.userManager.User(username)
if err != nil {
return mapUserErrorToHTTP(err)
} else if u.IsAdmin() && !isAdminSocketRequest(r) {
return errHTTPForbiddenAdminSocketOnly // Admin users' tokens can only be managed over the admin socket
}
users = []*user.User{u}
} else if !isAdminSocketRequest(r) {
return errHTTPForbiddenAdminSocketOnly // All users' tokens in one call, too much for a leaked admin token
} else {
var err error
if users, err = s.userManager.Users(); err != nil {
return err
}
}
response := make([]*apiUserTokenResponse, 0)
for _, u := range users {
tokens, err := s.userManager.Tokens(u.ID)
if err != nil {
return err
}
for _, t := range tokens {
response = append(response, newAPIUserTokenResponse(u.Name, t))
}
}
return s.writeJSON(w, response)
}
func (s *Server) handleUsersTokensAdd(w http.ResponseWriter, r *http.Request, v *visitor) error {
req, err := readJSONWithLimit[apiUsersTokensAddRequest](r.Body, jsonBodyBytesLimit, false)
if err != nil {
return err
} else if req.Username == user.Everyone {
return errHTTPBadRequest.Wrap("cannot create tokens for the everyone user")
}
u, err := s.userManager.User(req.Username)
if err != nil {
return mapUserErrorToHTTP(err)
} else if u.IsAdmin() && !isAdminSocketRequest(r) {
return errHTTPForbiddenAdminSocketOnly // Admin users' tokens can only be managed over the admin socket
}
expires := time.Unix(0, 0)
if req.Expires != 0 {
expires = time.Unix(req.Expires, 0)
if !expires.After(time.Now()) {
return errHTTPBadRequest.Wrap("expires must be in the future")
}
}
adminLog(v, r).Info("Creating token for user %s", req.Username)
token, err := s.userManager.CreateToken(u.ID, req.Label, expires, netip.IPv4Unspecified(), false)
if err != nil {
return mapUserErrorToHTTP(err)
}
return s.writeJSON(w, newAPIUserTokenResponse(u.Name, token))
}
func (s *Server) handleUsersTokensDelete(w http.ResponseWriter, r *http.Request, v *visitor) error {
req, err := readJSONWithLimit[apiUsersTokensDeleteRequest](r.Body, jsonBodyBytesLimit, false)
if err != nil {
return err
}
if err := s.userManager.ResetAccess(req.Username, req.Topic); err != nil {
u, err := s.userManager.User(req.Username)
if err != nil {
return mapUserErrorToHTTP(err)
} else if u.IsAdmin() && !isAdminSocketRequest(r) {
return errHTTPForbiddenAdminSocketOnly // Admin users' tokens can only be managed over the admin socket
}
adminLog(v, r).Info("Removing token for user %s", req.Username)
if err := s.userManager.RemoveToken(u.ID, req.Token); err != nil {
return mapUserErrorToHTTP(err)
}
return s.writeJSON(w, newSuccessResponse())
}
func (s *Server) handleUsersPasswordReset(w http.ResponseWriter, r *http.Request, v *visitor) error {
req, err := readJSONWithLimit[apiUsersPasswordResetRequest](r.Body, jsonBodyBytesLimit, false)
if err != nil {
return err
} else if s.config.BaseURL == "" {
return errHTTPBadRequestBaseURLNotConfigured
}
u, err := s.userManager.User(req.Username)
if err != nil {
return mapUserErrorToHTTP(err)
} else if u.Provisioned {
return errHTTPConflictProvisionedUserChange
} else if u.IsAdmin() && !isAdminSocketRequest(r) {
return errHTTPForbiddenAdminSocketOnly
}
var email string
if req.SendEmail {
if s.mailer == nil {
return errHTTPBadRequest.Wrap("smtp-sender-addr is not configured on the server")
}
if email, err = s.userManager.PrimaryEmail(u.ID); err != nil {
return err
} else if email == "" {
return errHTTPBadRequestUserNoPrimaryEmail
}
}
token, err := s.userManager.AddMagicLink(user.MagicLinkKindPasswordReset, u.ID, "", passwordResetTokenExpiry)
if err != nil {
return err
}
if err := s.killUserSubscriber(u, req.Topic); err != nil { // This may be a pattern
link := s.config.BaseURL + webAppPasswordResetPathPrefix + token
adminLog(v, r).Info("Creating password reset link for user %s", req.Username)
if req.SendEmail {
if err := s.mailer.SendPasswordReset(email, link); err != nil {
adminLog(v, r).Err(err).Warn("Failed to send password reset email to %s", email)
return fmt.Errorf("failed to send reset email to %s: %w", email, err)
}
}
return s.writeJSON(w, &apiUsersPasswordResetResponse{
Link: link,
Email: email,
})
}
func (s *Server) handleAdminTiersGet(w http.ResponseWriter, r *http.Request, v *visitor) error {
tiers, err := s.userManager.Tiers()
if err != nil {
return err
}
counts, err := s.userManager.TierUsersCount()
if err != nil {
return err
}
slices.SortFunc(tiers, func(a, b *user.Tier) int {
return strings.Compare(a.Code, b.Code)
})
response := make([]*apiAdminTier, len(tiers))
for i, t := range tiers {
response[i] = &apiAdminTier{
ID: t.ID,
Code: t.Code,
Name: t.Name,
MessagesLimit: t.MessageLimit,
MessagesExpiryDuration: int64(t.MessageExpiryDuration.Seconds()),
EmailsLimit: t.EmailLimit,
CallsLimit: t.CallLimit,
ReservationsLimit: t.ReservationLimit,
AttachmentFileSizeLimit: t.AttachmentFileSizeLimit,
AttachmentTotalSizeLimit: t.AttachmentTotalSizeLimit,
AttachmentExpiryDuration: int64(t.AttachmentExpiryDuration.Seconds()),
AttachmentBandwidthLimit: t.AttachmentBandwidthLimit,
StripeMonthlyPriceID: t.StripeMonthlyPriceID,
StripeYearlyPriceID: t.StripeYearlyPriceID,
Users: counts[t.Code],
}
}
return s.writeJSON(w, response)
}
func (s *Server) handleAdminTiersAdd(w http.ResponseWriter, r *http.Request, v *visitor) error {
tier, err := readAdminTierRequest(r)
if err != nil {
return err
}
if _, err := s.userManager.Tier(tier.Code); err == nil {
return errHTTPConflictTierExists
} else if !errors.Is(err, user.ErrTierNotFound) {
return err
}
adminLog(v, r).Info("Adding tier %s", tier.Code)
if err := s.userManager.AddTier(tier); err != nil {
return mapUserErrorToHTTP(err)
}
return s.writeJSON(w, newSuccessResponse())
}
func (s *Server) handleAdminTiersUpdate(w http.ResponseWriter, r *http.Request, v *visitor) error {
tier, err := readAdminTierRequest(r)
if err != nil {
return err
}
if _, err := s.userManager.Tier(tier.Code); err != nil {
return mapUserErrorToHTTP(err)
}
adminLog(v, r).Info("Updating tier %s", tier.Code)
if err := s.userManager.UpdateTier(tier); err != nil {
return mapUserErrorToHTTP(err)
}
return s.writeJSON(w, newSuccessResponse())
}
func (s *Server) handleAdminTiersDelete(w http.ResponseWriter, r *http.Request, v *visitor) error {
req, err := readJSONWithLimit[apiAdminTierDeleteRequest](r.Body, jsonBodyBytesLimit, false)
if err != nil {
return err
}
if _, err := s.userManager.Tier(req.Code); err != nil {
return mapUserErrorToHTTP(err)
}
counts, err := s.userManager.TierUsersCount()
if err != nil {
return err
} else if counts[req.Code] > 0 {
return errHTTPConflictTierInUse
}
adminLog(v, r).Info("Removing tier %s", req.Code)
if err := s.userManager.RemoveTier(req.Code); err != nil {
return mapUserErrorToHTTP(err)
}
return s.writeJSON(w, newSuccessResponse())
}
@@ -201,3 +473,187 @@ func (s *Server) killUserSubscriber(u *user.User, topicPattern string) error {
}
return nil
}
// broadcastAccessChanged tells the other cluster nodes that this node changed access entries, so
// they reload their access cache, which ends the streams that lost access there (see
// reauthorizeSubscribers). The user manager calls it after every access write. The signal is
// best effort: the periodic reload catches a lost one.
func (s *Server) broadcastAccessChanged() {
s.cluster.BroadcastState(&cluster.State{
AccessChanged: true,
})
}
// handlePeerAccessChanged has the access cache reloaded after a peer changed access entries, never
// re-broadcasting it (loop prevention)
func (s *Server) handlePeerAccessChanged() {
if s.userManager == nil {
return
}
s.userManager.RequestAccessCacheReload() // Coalesced: a burst of peer changes costs at most two reloads
}
// userDeleted ends a deleted user's streams on this node and tells the other cluster nodes, which end
// theirs (see handleUsersDeleted). Called by both deletes: by an admin, and of one's own account.
func (s *Server) userDeleted(u *user.User) {
s.handleUsersDeleted([]string{u.ID})
s.cluster.BroadcastState(&cluster.State{
DeletedUserIDs: []string{u.ID},
})
}
// handleUsersDeleted ends all streams of the given (deleted) users on this node, on every topic,
// after a local delete or a peer's, never re-broadcasting. An access cache reload does not find
// them: a user without access entries (e.g. an admin) leaves nothing in the cache to compare.
func (s *Server) handleUsersDeleted(userIDs []string) {
for _, t := range s.allTopics() {
for _, userID := range userIDs {
t.CancelSubscriberUser(userID) // An empty user ID cancels nobody
}
}
}
// reauthorizeSubscribers ends the subscriptions that changed access entries no longer allow, checking
// only matching topics. The user manager calls it after an access cache reload that found changes.
// It authorizes each subscription's own user as of subscribing, against the cache, without a
// database query; a deleted user's streams end through handleUsersDeleted instead.
func (s *Server) reauthorizeSubscribers(changes []*user.AccessChange) {
for _, change := range changes {
for _, t := range s.topicsForAccessChange(change.TopicPattern) {
t.CancelSubscribersWhere(func(u *user.User) bool {
if change.Username != user.Everyone && (u == nil || u.Name != change.Username) {
return false // The entry does not apply to this subscriber
}
return s.userManager.Authorize(u, t.ID, user.PermissionRead) != nil
})
}
}
}
// authorizeSubscribed authorizes a new stream once more, right after it subscribed. An access change
// whose re-check (see reauthorizeSubscribers) ran between the request's authorization and its
// subscribing would otherwise miss the stream for good. Each check sees the other: the re-check runs
// after the cache was swapped, so either it finds the subscription or this check finds the new
// entries. Only with the access cache, where it is a lookup in memory and re-checks happen at all.
func (s *Server) authorizeSubscribed(v *visitor, topics []*topic) error {
if s.userManager == nil || !s.config.AuthAccessCacheEnabled {
return nil
}
for _, t := range topics {
if err := s.userManager.Authorize(v.User(), t.ID, user.PermissionRead); err != nil {
return errHTTPForbidden
}
}
return nil
}
// topicsForAccessChange returns the local topics a changed access entry's topic or pattern matches
func (s *Server) topicsForAccessChange(topicPattern string) []*topic {
if !strings.Contains(topicPattern, "*") {
s.mu.RLock()
defer s.mu.RUnlock()
if t, ok := s.topics[topicPattern]; ok {
return []*topic{t}
}
return nil
}
topics, err := s.topicsFromPattern(topicPattern)
if err != nil {
log.Tag(tagSubscribe).Err(err).Warn("Cannot match topics against access pattern %s", topicPattern)
return nil
}
return topics
}
// checkRoleChange validates a requested role change. Roles can only be set over the admin socket.
func checkRoleChange(r *http.Request, role string) (user.Role, error) {
if !isAdminSocketRequest(r) {
return "", errHTTPForbiddenAdminSocketOnly
} else if !user.AllowedRole(user.Role(role)) {
return "", errHTTPBadRequestRoleInvalid
}
return user.Role(role), nil
}
// readAdminTierRequest reads and validates an apiAdminTier request, and converts it to a user.Tier
func readAdminTierRequest(r *http.Request) (*user.Tier, error) {
req, err := readJSONWithLimit[apiAdminTier](r.Body, jsonBodyBytesLimit, false)
if err != nil {
return nil, err
} else if !user.AllowedTier(req.Code) || strings.TrimSpace(req.Name) == "" {
return nil, errHTTPBadRequestTierCodeInvalid
}
return &user.Tier{
Code: req.Code,
Name: strings.TrimSpace(req.Name),
MessageLimit: req.MessagesLimit,
MessageExpiryDuration: time.Duration(req.MessagesExpiryDuration) * time.Second,
EmailLimit: req.EmailsLimit,
CallLimit: req.CallsLimit,
ReservationLimit: req.ReservationsLimit,
AttachmentFileSizeLimit: req.AttachmentFileSizeLimit,
AttachmentTotalSizeLimit: req.AttachmentTotalSizeLimit,
AttachmentExpiryDuration: time.Duration(req.AttachmentExpiryDuration) * time.Second,
AttachmentBandwidthLimit: req.AttachmentBandwidthLimit,
StripeMonthlyPriceID: req.StripeMonthlyPriceID,
StripeYearlyPriceID: req.StripeYearlyPriceID,
}, nil
}
func newAPIUserTokenResponse(username string, t *user.Token) *apiUserTokenResponse {
response := &apiUserTokenResponse{
Username: username,
Token: t.Value,
Label: t.Label,
Expires: t.Expires.Unix(),
Provisioned: t.Provisioned,
}
// Tokens created without a request (admin API, CLI, provisioning) carry their creation time and
// origin 0.0.0.0 until first used; report those as never used
if t.LastOrigin.IsValid() && !t.LastOrigin.IsUnspecified() {
response.LastAccess = t.LastAccess.Unix()
response.LastOrigin = t.LastOrigin.String()
}
return response
}
// mapUserErrorToHTTP maps user manager errors to HTTP errors
func mapUserErrorToHTTP(err error) error {
switch {
case errors.Is(err, user.ErrUserNotFound):
return errHTTPBadRequestUserNotFound
case errors.Is(err, user.ErrUserExists):
return errHTTPConflictUserExists
case errors.Is(err, user.ErrTierNotFound):
return errHTTPBadRequestTierInvalid
case errors.Is(err, user.ErrTokenNotFound):
return errHTTPBadRequestTokenNotFound
case errors.Is(err, user.ErrProvisionedUserChange):
return errHTTPConflictProvisionedUserChange
case errors.Is(err, user.ErrProvisionedTokenChange):
return errHTTPConflictProvisionedTokenChange
case errors.Is(err, user.ErrTooManyReservations):
return errHTTPBadRequestTierReservationsLimit
case errors.Is(err, user.ErrPasswordHashInvalid), errors.Is(err, user.ErrPasswordHashWeak):
return errHTTPBadRequestPasswordHashInvalid
case errors.Is(err, user.ErrInvalidArgument):
return errHTTPBadRequest
}
return err
}
// isAdminSocketRequest reports whether the request came in over the local admin socket. Those
// requests may do what the network API refuses: change roles, and change or delete admins.
func isAdminSocketRequest(r *http.Request) bool {
socket, err := fromContext[bool](r, contextAdminSocket)
return err == nil && socket
}
// adminLog returns the audit log event for a mutating admin request, with how it arrived
func adminLog(v *visitor, r *http.Request) *log.Event {
via := "http"
if isAdminSocketRequest(r) {
via = "socket"
}
return logvr(v, r).Tag(tagAdmin).Field("admin_via", via)
}
server/server_admin_socket.go
+48
View File
@@ -0,0 +1,48 @@
package server
import (
"net/http"
"net/netip"
"slices"
"heckel.io/ntfy/v2/user"
)
var (
// adminSocketPaths are the paths served over the admin socket. All of them are admin-only
// routes in handleInternal (ensureAdmin), which the socket's synthetic admin user passes.
adminSocketPaths = []string{
apiVersionPath,
apiUsersPath,
apiUsersAccessPath,
apiUsersTokensPath,
apiUsersPasswordResetPath,
apiTiersPath,
}
// adminSocketUser is the synthetic admin every socket request runs as. Its name is one that
// user.AllowedUsername rejects, so it can never collide with a real account.
adminSocketUser = &user.User{
ID: "u_admin_socket",
Name: "(admin socket)",
Role: user.RoleAdmin,
Stats: &user.Stats{}, // Read by newVisitor
Billing: &user.Billing{}, // Read by the visitor's log context
}
)
// handleAdminSocket serves one admin API request that arrived over the local socket. There is
// no authentication: whoever can open the socket (file mode 0660, owner ntfy) is an admin.
// The request runs as adminSocketUser, bypasses rate limiting and the ban feed, and carries
// contextAdminSocket so handlers can allow what the network API refuses.
func (s *Server) handleAdminSocket(w http.ResponseWriter, r *http.Request) {
r = withContext(r, map[contextKey]any{contextAdminSocket: true})
v := newVisitor(s.visitors, netip.AddrFrom4([4]byte{127, 0, 0, 1}), adminSocketUser)
if !slices.Contains(adminSocketPaths, r.URL.Path) {
s.handleError(w, r, v, errHTTPNotFound) // Publishing, subscribing, the web app, etc. are not served here
return
}
if err := s.handleInternal(w, r, v); err != nil {
s.handleError(w, r, v, err)
}
}
server/server_admin_test.go
+825
View File
@@ -2,9 +2,16 @@ package server
import (
"encoding/json"
"fmt"
"github.com/stretchr/testify/require"
dbtest "heckel.io/ntfy/v2/db/test"
"heckel.io/ntfy/v2/model"
"heckel.io/ntfy/v2/user"
"heckel.io/ntfy/v2/util"
"net/http"
"net/http/httptest"
"net/netip"
"strings"
"sync/atomic"
"testing"
"time"
@@ -364,6 +371,39 @@ func TestUser_AddRemove_Failures(t *testing.T) {
})
}
func TestUser_ChangeRemoveProvisionedUser(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
conf := newTestConfigWithAuthFile(t, databaseURL)
conf.AuthUsers = []*user.User{
{Name: "philuser", Hash: "$2a$10$U4WSIYY6evyGmZaraavM2e2JeVG6EMGUKN1uUwufUeeRd4Jpg6cGC", Role: user.RoleUser}, // philuser:philpass
}
s := newTestServer(t, conf)
defer s.closeDatabases()
// Create admin
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
// Cannot delete provisioned user
rr := request(t, s, "DELETE", "/v1/users", `{"username": "philuser"}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 409, rr.Code)
require.Equal(t, 40905, toHTTPError(t, rr.Body.String()).Code)
// Cannot change password of provisioned user
rr = request(t, s, "PUT", "/v1/users", `{"username": "philuser", "password": "newpass"}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 409, rr.Code)
require.Equal(t, 40905, toHTTPError(t, rr.Body.String()).Code)
// User is still there, with the old password
u, err := s.userManager.Authenticate("philuser", "philpass")
require.Nil(t, err)
require.Equal(t, "philuser", u.Name)
})
}
func TestAccess_AllowReset(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
c := newTestConfigWithAuthFile(t, databaseURL)
@@ -459,3 +499,788 @@ func TestAccess_AllowReset_KillConnection(t *testing.T) {
})
})
}
func TestAccess_WritesTellPeers(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
// Subscribers are per node: an access write tells the other nodes to reload their access
// cache (which ends the streams that lost access there). Deleting a user also asks them to
// cancel the user's streams, since a user without access entries (e.g. an admin) leaves
// nothing in the cache to find.
c := newTestConfigWithAuthFile(t, databaseURL)
s := newTestServer(t, c)
defer s.closeDatabases()
b := &fakeCluster{}
s.cluster = b
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
require.Nil(t, s.userManager.AllowAccess("ben", "gol*", user.PermissionRead))
ben, err := s.userManager.User("ben")
require.Nil(t, err)
admin := map[string]string{"Authorization": util.BasicAuth("phil", "phil")}
before := b.AccessChanged()
rr := request(t, s, "DELETE", "/v1/users/access", `{"username": "ben", "topic":"gol*"}`, admin)
require.Equal(t, 200, rr.Code)
require.Greater(t, b.AccessChanged(), before)
require.Empty(t, b.DeletedUsers())
rr = request(t, s, "DELETE", "/v1/users", `{"username": "ben"}`, admin)
require.Equal(t, 200, rr.Code)
require.Equal(t, []string{ben.ID}, b.DeletedUsers())
})
}
func TestAccess_PeerAccessChangedReloadsAndCancels(t *testing.T) {
// Told by a peer that access changed, this node reloads its access cache: the reload ends the
// subscriber who lost access, and the client's reconnect is denied, not let in by a stale entry
dsn := dbtest.CreateTestPostgresSchema(t)
newNode := func() *Server {
conf := newTestConfig(t, dsn)
conf.AuthDefault = user.PermissionDenyAll
conf.AuthAccessCacheEnabled = true
conf.AuthAccessCacheReloadInterval = time.Hour
return newTestServer(t, conf)
}
a := newNode()
require.Nil(t, a.userManager.AddUser("alice", "password", user.RoleUser, false))
require.Nil(t, a.userManager.AllowAccess("alice", "private", user.PermissionRead))
b := newNode()
headers := map[string]string{"Authorization": util.BasicAuth("alice", "password")}
stream := subscribeStream(t, b, "private", headers)
require.Nil(t, a.userManager.ResetAccess("alice", "private"))
b.handlePeerAccessChanged()
select {
case <-stream.ended:
case <-time.After(5 * time.Second):
t.Fatal("alice's stream was not canceled")
}
require.Equal(t, 403, request(t, b, "GET", "/private/json?poll=1", "", headers).Code)
}
func TestAccess_PeerUsersDeletedCancels(t *testing.T) {
// A peer deleted a user: this node cancels the user's streams
s := newTestServer(t, newTestConfigWithAuthFile(t, ""))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleAdmin, false))
ben, err := s.userManager.User("ben")
require.Nil(t, err)
stream := subscribeStream(t, s, "mytopic", map[string]string{"Authorization": util.BasicAuth("ben", "ben")})
s.handleUsersDeleted([]string{ben.ID})
select {
case <-stream.ended:
case <-time.After(5 * time.Second):
t.Fatal("ben's stream was not canceled")
}
}
func TestAccess_ExternalRevocationCaughtByAccessReload(t *testing.T) {
// An access change made elsewhere (another node whose relay got lost, or the ntfy access CLI)
// reaches node B at its next access cache reload. B ends the streams of the user who lost
// access, and only those: carol, still allowed on the same topic, keeps her stream.
dsn := dbtest.CreateTestPostgresSchema(t)
newNode := func() *Server {
conf := newTestConfig(t, dsn)
conf.AuthDefault = user.PermissionDenyAll
conf.AuthAccessCacheEnabled = true
conf.AuthAccessCacheReloadInterval = 500 * time.Millisecond
return newTestServer(t, conf)
}
a := newNode()
for _, name := range []string{"alice", "carol"} {
require.Nil(t, a.userManager.AddUser(name, "password", user.RoleUser, false))
require.Nil(t, a.userManager.AllowAccess(name, "private", user.PermissionRead))
}
b := newNode()
alice := map[string]string{"Authorization": util.BasicAuth("alice", "password")}
carol := map[string]string{"Authorization": util.BasicAuth("carol", "password")}
aliceStream := subscribeStream(t, b, "private", alice)
carolStream := subscribeStream(t, b, "private", carol)
require.Nil(t, a.userManager.ResetAccess("alice", "private"))
select {
case <-aliceStream.ended:
case <-time.After(5 * time.Second):
t.Fatal("alice's stream on node B was not canceled by its access cache reload")
}
require.Equal(t, 403, request(t, b, "GET", "/private/json?poll=1", "", alice).Code)
time.Sleep(time.Second) // Two more reloads
select {
case <-carolStream.ended:
t.Fatal("carol's stream was canceled, but she can still read the topic")
default:
}
}
func TestAccess_RecheckUsesEachSubscriptionsOwnUser(t *testing.T) {
// Two streams of one user can carry different users as of subscribing, e.g. one opened before a
// demotion (admin) and one after (user). A revoke is checked against each stream's own user: the
// regular stream ends, the admin one stays (an accepted limit until it reconnects).
conf := newTestConfigWithAuthFile(t, "")
conf.AuthDefault = user.PermissionDenyAll
conf.AuthAccessCacheEnabled = true
conf.AuthAccessCacheReloadInterval = time.Hour
s := newTestServer(t, conf)
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
require.Nil(t, s.userManager.AllowAccess("ben", "private", user.PermissionRead))
ben, err := s.userManager.User("ben")
require.Nil(t, err)
benAsAdmin := &user.User{
ID: ben.ID,
Name: ben.Name,
Role: user.RoleAdmin,
}
topics, err := s.topicsFromIDs(nil, "private")
require.Nil(t, err)
var adminCanceled, userCanceled atomic.Bool
noop := func(*visitor, *model.Message) error {
return nil
}
topics[0].Subscribe(noop, benAsAdmin, func() {
adminCanceled.Store(true)
})
topics[0].Subscribe(noop, ben, func() {
userCanceled.Store(true)
})
require.Nil(t, s.userManager.ResetAccess("ben", "private")) // Reloads the cache and re-checks
require.True(t, userCanceled.Load())
require.False(t, adminCanceled.Load())
}
func TestAccess_RevokeBetweenAuthorizeAndSubscribe(t *testing.T) {
// A revoke, and its re-check of live subscribers, can land after a request was authorized but
// before its stream subscribed: no later re-check would find the stream, so it must not slip through
conf := newTestConfigWithAuthFile(t, "")
conf.AuthDefault = user.PermissionDenyAll
conf.AuthAccessCacheEnabled = true
conf.AuthAccessCacheReloadInterval = time.Hour
s := newTestServer(t, conf)
require.Nil(t, s.userManager.AddUser("alice", "alice", user.RoleUser, false))
require.Nil(t, s.userManager.AllowAccess("alice", "private", user.PermissionRead))
testHookBeforeSubscribe = func() {
require.Nil(t, s.userManager.ResetAccess("alice", "private"))
}
t.Cleanup(func() {
testHookBeforeSubscribe = nil
})
done := make(chan int)
go func() {
rr := request(t, s, "GET", "/private/json", "", map[string]string{
"Authorization": util.BasicAuth("alice", "alice"),
})
done <- rr.Code
}()
select {
case code := <-done:
require.Equal(t, 403, code)
case <-time.After(3 * time.Second):
t.Fatal("the stream subscribed after the revoke stayed open")
}
}
func TestUser_GetWithDetails(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
conf := newTestConfigWithAuthFile(t, databaseURL)
conf.AuthDefault = user.PermissionRead
conf.AuthUsers = []*user.User{
{Name: "philuser", Hash: "$2a$10$U4WSIYY6evyGmZaraavM2e2JeVG6EMGUKN1uUwufUeeRd4Jpg6cGC", Role: user.RoleUser}, // philuser:philpass
}
conf.AuthAccess = map[string][]*user.Grant{
"philuser": {{TopicPattern: "provisioned-topic", Permission: user.PermissionReadWrite}},
}
s := newTestServer(t, conf)
defer s.closeDatabases()
// Admin, a user with a tier and a manual grant
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddTier(&user.Tier{Code: "pro", Name: "Pro"}))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
require.Nil(t, s.userManager.ChangeTier("ben", "pro"))
require.Nil(t, s.userManager.AllowAccess("ben", "bens-topic", user.PermissionRead))
rr := request(t, s, "GET", "/v1/users", "", map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 200, rr.Code)
var users []*apiUserResponse
require.Nil(t, json.NewDecoder(rr.Body).Decode(&users))
byName := make(map[string]*apiUserResponse)
for _, u := range users {
byName[u.Username] = u
}
// Tier code and display name
require.Equal(t, "pro", byName["ben"].Tier)
require.Equal(t, "Pro", byName["ben"].TierName)
require.False(t, byName["ben"].Provisioned)
require.Equal(t, []*apiUserGrantResponse{{Topic: "bens-topic", Permission: "read-only"}}, byName["ben"].Grants)
// Provisioned user and grant
require.True(t, byName["philuser"].Provisioned)
require.Equal(t, []*apiUserGrantResponse{{Topic: "provisioned-topic", Permission: "read-write", Provisioned: true}}, byName["philuser"].Grants)
// Default access only on the everyone entry
require.Equal(t, "read-only", byName[user.Everyone].DefaultAccess)
require.Equal(t, "", byName["ben"].DefaultAccess)
require.Equal(t, "", byName["phil"].DefaultAccess)
})
}
func TestUser_AddWithRole(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
// Role over HTTP is refused, even for admins
rr := request(t, s, "POST", "/v1/users", `{"username": "ben", "password":"ben", "role":"admin"}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 403, rr.Code)
require.Equal(t, 40302, toHTTPError(t, rr.Body.String()).Code)
_, err := s.userManager.User("ben")
require.ErrorIs(t, err, user.ErrUserNotFound)
// Invalid role over the socket
rr = requestAdminSocket(t, s, "POST", "/v1/users", `{"username": "ben", "password":"ben", "role":"superuser"}`)
require.Equal(t, 400, rr.Code)
require.Equal(t, 40066, toHTTPError(t, rr.Body.String()).Code)
// Role over the socket works
rr = requestAdminSocket(t, s, "POST", "/v1/users", `{"username": "ben", "password":"ben", "role":"admin"}`)
require.Equal(t, 200, rr.Code)
u, err := s.userManager.User("ben")
require.Nil(t, err)
require.Equal(t, user.RoleAdmin, u.Role)
})
}
func TestUser_AddWithInvalidHash(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
rr := request(t, s, "POST", "/v1/users", `{"username": "ben", "hash":"not-a-bcrypt-hash"}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 400, rr.Code)
require.Equal(t, 40069, toHTTPError(t, rr.Body.String()).Code)
// The error names the JSON fields as they are spelled
rr = request(t, s, "POST", "/v1/users", `{"username": "ben"}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 400, rr.Code)
require.Contains(t, toHTTPError(t, rr.Body.String()).Message, `"password" or "hash" missing`)
rr = request(t, s, "PUT", "/v1/users", `{"username": "ben"}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 400, rr.Code)
require.Contains(t, toHTTPError(t, rr.Body.String()).Message, `"password", "hash", "tier" or "role"`)
})
}
func TestUser_ChangeTierReset(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddTier(&user.Tier{Code: "pro", Name: "Pro"}))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
require.Nil(t, s.userManager.ChangeTier("ben", "pro"))
rr := request(t, s, "PUT", "/v1/users", `{"username": "ben", "tier":"-"}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 200, rr.Code)
u, err := s.userManager.User("ben")
require.Nil(t, err)
require.Nil(t, u.Tier)
})
}
func TestUser_ChangeTierTooManyReservations(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddTier(&user.Tier{Code: "big", Name: "Big", ReservationLimit: 5}))
require.Nil(t, s.userManager.AddTier(&user.Tier{Code: "small", Name: "Small", ReservationLimit: 1}))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
require.Nil(t, s.userManager.ChangeTier("ben", "big"))
require.Nil(t, s.userManager.AddReservation("ben", "topic1", user.PermissionDenyAll, 5))
require.Nil(t, s.userManager.AddReservation("ben", "topic2", user.PermissionDenyAll, 5))
rr := request(t, s, "PUT", "/v1/users", `{"username": "ben", "tier":"small"}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 400, rr.Code)
require.Equal(t, 40063, toHTTPError(t, rr.Body.String()).Code)
})
}
func TestUser_ChangeRole(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
// Refused over HTTP
rr := request(t, s, "PUT", "/v1/users", `{"username": "ben", "role":"admin"}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 403, rr.Code)
require.Equal(t, 40302, toHTTPError(t, rr.Body.String()).Code)
// Invalid role, and missing user over the socket
rr = requestAdminSocket(t, s, "PUT", "/v1/users", `{"username": "ben", "role":"superuser"}`)
require.Equal(t, 40066, toHTTPError(t, rr.Body.String()).Code)
rr = requestAdminSocket(t, s, "PUT", "/v1/users", `{"username": "emma", "role":"admin"}`)
require.Equal(t, 40031, toHTTPError(t, rr.Body.String()).Code)
_, err := s.userManager.User("emma")
require.ErrorIs(t, err, user.ErrUserNotFound)
// Promote and demote over the socket
rr = requestAdminSocket(t, s, "PUT", "/v1/users", `{"username": "ben", "role":"admin"}`)
require.Equal(t, 200, rr.Code)
u, err := s.userManager.User("ben")
require.Nil(t, err)
require.Equal(t, user.RoleAdmin, u.Role)
rr = requestAdminSocket(t, s, "PUT", "/v1/users", `{"username": "ben", "role":"user"}`)
require.Equal(t, 200, rr.Code)
u, err = s.userManager.User("ben")
require.Nil(t, err)
require.Equal(t, user.RoleUser, u.Role)
})
}
func TestUser_ChangeAdminPasswordOverSocket(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("admin", "admin", user.RoleAdmin, false))
rr := requestAdminSocket(t, s, "PUT", "/v1/users", `{"username": "admin", "password": "admin-new"}`)
require.Equal(t, 200, rr.Code)
_, err := s.userManager.Authenticate("admin", "admin-new")
require.Nil(t, err)
})
}
func TestUser_DeleteAdmin(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddUser("admin", "admin", user.RoleAdmin, false))
// Refused over HTTP
rr := request(t, s, "DELETE", "/v1/users", `{"username": "admin"}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 401, rr.Code)
// Allowed over the socket
rr = requestAdminSocket(t, s, "DELETE", "/v1/users", `{"username": "admin"}`)
require.Equal(t, 200, rr.Code)
_, err := s.userManager.User("admin")
require.ErrorIs(t, err, user.ErrUserNotFound)
})
}
func TestUser_DeleteRemovesWebPushSubscriptions(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, configureAuth(t, newTestConfigWithWebPush(t, databaseURL)))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
u, err := s.userManager.User("ben")
require.Nil(t, err)
require.Nil(t, s.webPush.UpsertSubscription(testWebPushEndpoint, "kSC3T8aN1JCQxxPdrFLrZg", "BMKKbxdUU_xLS7G1Wh5AN8PvWOjCzkCuKZYb8apcqYrDxjOF_2piggBnoJLQYx9IeSD70fNuwawI3e9Y8m3S3PE", u.ID, netip.MustParseAddr("1.2.3.4"), []string{"test-topic"}))
requireSubscriptionCount(t, s, "test-topic", 1)
rr := requestAdminSocket(t, s, "DELETE", "/v1/users", `{"username": "ben"}`)
require.Equal(t, 200, rr.Code)
requireSubscriptionCount(t, s, "test-topic", 0)
})
}
func TestAccess_AllowAdminUser(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
rr := requestAdminSocket(t, s, "PUT", "/v1/users/access", `{"username": "phil", "topic":"gold", "permission":"ro"}`)
require.Equal(t, 400, rr.Code)
require.Equal(t, 40064, toHTTPError(t, rr.Body.String()).Code)
})
}
func TestAccess_ResetAll(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
require.Nil(t, s.userManager.AddUser("emma", "emma", user.RoleUser, false))
require.Nil(t, s.userManager.AllowAccess("ben", "gold", user.PermissionRead))
require.Nil(t, s.userManager.AllowAccess("emma", "silver", user.PermissionRead))
require.Nil(t, s.userManager.AllowAccess(user.Everyone, "public", user.PermissionRead))
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
// Over HTTP, resetting the whole ACL is refused, even for admins
rr := request(t, s, "DELETE", "/v1/users/access", `{}`, map[string]string{
"Authorization": util.BasicAuth("phil", "phil"),
})
require.Equal(t, 403, rr.Code)
require.Equal(t, 40302, toHTTPError(t, rr.Body.String()).Code)
grants, err := s.userManager.AllGrants()
require.Nil(t, err)
require.Len(t, grants, 3)
// Over the socket, it works
rr = requestAdminSocket(t, s, "DELETE", "/v1/users/access", `{}`)
require.Equal(t, 200, rr.Code)
grants, err = s.userManager.AllGrants()
require.Nil(t, err)
require.Empty(t, grants)
})
}
func TestAccess_ResetUserAllTopics(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
require.Nil(t, s.userManager.AddUser("emma", "emma", user.RoleUser, false))
require.Nil(t, s.userManager.AllowAccess("ben", "gold", user.PermissionRead))
require.Nil(t, s.userManager.AllowAccess("ben", "silver", user.PermissionRead))
require.Nil(t, s.userManager.AllowAccess("emma", "silver", user.PermissionRead))
rr := requestAdminSocket(t, s, "DELETE", "/v1/users/access", `{"username":"ben"}`)
require.Equal(t, 200, rr.Code)
bensGrants, err := s.userManager.Grants("ben")
require.Nil(t, err)
require.Empty(t, bensGrants)
emmasGrants, err := s.userManager.Grants("emma")
require.Nil(t, err)
require.Len(t, emmasGrants, 1)
// Unknown user
rr = requestAdminSocket(t, s, "DELETE", "/v1/users/access", `{"username":"nobody"}`)
require.Equal(t, 40031, toHTTPError(t, rr.Body.String()).Code)
})
}
func TestTokens_AddListRemove(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
require.Nil(t, s.userManager.AddUser("emma", "emma", user.RoleUser, false))
auth := map[string]string{"Authorization": util.BasicAuth("phil", "phil")}
// No tokens yet: empty array, not null
rr := request(t, s, "GET", "/v1/users/tokens?username=ben", "", auth)
require.Equal(t, 200, rr.Code)
require.Equal(t, "[]", strings.TrimSpace(rr.Body.String()))
// Add a token that never expires, and one that does
rr = request(t, s, "POST", "/v1/users/tokens", `{"username":"ben","label":"forever"}`, auth)
require.Equal(t, 200, rr.Code)
var forever apiUserTokenResponse
require.Nil(t, json.NewDecoder(rr.Body).Decode(&forever))
require.Equal(t, "ben", forever.Username)
require.Equal(t, "forever", forever.Label)
require.Equal(t, int64(0), forever.Expires)
require.True(t, user.ValidToken(forever.Token))
require.Equal(t, int64(0), forever.LastAccess) // Never used
require.Equal(t, "", forever.LastOrigin)
expires := time.Now().Add(time.Hour).Unix()
rr = request(t, s, "POST", "/v1/users/tokens", fmt.Sprintf(`{"username":"emma","expires":%d}`, expires), auth)
require.Equal(t, 200, rr.Code)
var expiring apiUserTokenResponse
require.Nil(t, json.NewDecoder(rr.Body).Decode(&expiring))
require.Equal(t, expires, expiring.Expires)
// The new token works
rr = request(t, s, "GET", "/v1/account", "", map[string]string{"Authorization": util.BearerAuth(forever.Token)})
require.Equal(t, 200, rr.Code)
// Listing all users' tokens is refused over HTTP, but works over the socket
rr = request(t, s, "GET", "/v1/users/tokens", "", auth)
require.Equal(t, 403, rr.Code)
require.Equal(t, 40302, toHTTPError(t, rr.Body.String()).Code)
rr = requestAdminSocket(t, s, "GET", "/v1/users/tokens", "")
require.Equal(t, 200, rr.Code)
var tokens []*apiUserTokenResponse
require.Nil(t, json.NewDecoder(rr.Body).Decode(&tokens))
require.Len(t, tokens, 2)
// List one user
rr = request(t, s, "GET", "/v1/users/tokens?username=ben", "", auth)
require.Equal(t, 200, rr.Code)
tokens = nil
require.Nil(t, json.NewDecoder(rr.Body).Decode(&tokens))
require.Len(t, tokens, 1)
require.Equal(t, forever.Token, tokens[0].Token)
// Remove
rr = request(t, s, "DELETE", "/v1/users/tokens", fmt.Sprintf(`{"username":"ben","token":"%s"}`, forever.Token), auth)
require.Equal(t, 200, rr.Code)
bensTokens, err := s.userManager.Tokens(mustUserID(t, s, "ben"))
require.Nil(t, err)
require.Empty(t, bensTokens)
})
}
func TestTokens_Failures(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
conf := newTestConfigWithAuthFile(t, databaseURL)
conf.AuthUsers = []*user.User{
{Name: "philuser", Hash: "$2a$10$U4WSIYY6evyGmZaraavM2e2JeVG6EMGUKN1uUwufUeeRd4Jpg6cGC", Role: user.RoleUser}, // philuser:philpass
}
conf.AuthTokens = map[string][]*user.Token{
"philuser": {{Value: "tk_op56p8lz5bf3cxkz9je99v9oc37lo", Label: "provisioned"}},
}
s := newTestServer(t, conf)
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
auth := map[string]string{"Authorization": util.BasicAuth("phil", "phil")}
// Expiry in the past
rr := request(t, s, "POST", "/v1/users/tokens", fmt.Sprintf(`{"username":"ben","expires":%d}`, time.Now().Add(-time.Hour).Unix()), auth)
require.Equal(t, 400, rr.Code)
// Unknown user, everyone
rr = request(t, s, "POST", "/v1/users/tokens", `{"username":"nobody"}`, auth)
require.Equal(t, 40031, toHTTPError(t, rr.Body.String()).Code)
rr = request(t, s, "POST", "/v1/users/tokens", `{"username":"*"}`, auth)
require.Equal(t, 400, rr.Code)
rr = request(t, s, "GET", "/v1/users/tokens?username=nobody", "", auth)
require.Equal(t, 40031, toHTTPError(t, rr.Body.String()).Code)
// Unknown token, provisioned token
rr = request(t, s, "DELETE", "/v1/users/tokens", `{"username":"ben","token":"tk_doesnotexistdoesnotexist12345"}`, auth)
require.Equal(t, 40065, toHTTPError(t, rr.Body.String()).Code)
rr = request(t, s, "DELETE", "/v1/users/tokens", `{"username":"philuser","token":"tk_op56p8lz5bf3cxkz9je99v9oc37lo"}`, auth)
require.Equal(t, 40906, toHTTPError(t, rr.Body.String()).Code)
// Non-admins cannot use the API
rr = request(t, s, "GET", "/v1/users/tokens?username=ben", "", map[string]string{"Authorization": util.BasicAuth("ben", "ben")})
require.Equal(t, 401, rr.Code)
})
}
func TestTokens_AdminTokensOnlyOverSocket(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
_, err := s.userManager.CreateToken(mustUserID(t, s, "phil"), "admin token", time.Unix(0, 0), netip.IPv4Unspecified(), false)
require.Nil(t, err)
_, err = s.userManager.CreateToken(mustUserID(t, s, "ben"), "user token", time.Unix(0, 0), netip.IPv4Unspecified(), false)
require.Nil(t, err)
auth := map[string]string{"Authorization": util.BasicAuth("phil", "phil")}
// Over HTTP, admin tokens cannot be listed, created or removed
rr := request(t, s, "GET", "/v1/users/tokens?username=phil", "", auth)
require.Equal(t, 403, rr.Code)
require.Equal(t, 40302, toHTTPError(t, rr.Body.String()).Code)
rr = request(t, s, "GET", "/v1/users/tokens?username=ben", "", auth)
require.Equal(t, 200, rr.Code)
var tokens []*apiUserTokenResponse
require.Nil(t, json.NewDecoder(rr.Body).Decode(&tokens))
require.Len(t, tokens, 1)
require.Equal(t, "ben", tokens[0].Username)
rr = request(t, s, "POST", "/v1/users/tokens", `{"username":"phil"}`, auth)
require.Equal(t, 403, rr.Code)
require.Equal(t, 40302, toHTTPError(t, rr.Body.String()).Code)
// Over the socket, they are visible and can be created
rr = requestAdminSocket(t, s, "GET", "/v1/users/tokens", "")
require.Equal(t, 200, rr.Code)
tokens = nil
require.Nil(t, json.NewDecoder(rr.Body).Decode(&tokens))
require.Len(t, tokens, 2)
rr = requestAdminSocket(t, s, "POST", "/v1/users/tokens", `{"username":"phil"}`)
require.Equal(t, 200, rr.Code)
})
}
func TestPasswordReset_Link(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s, mailer, _ := newEmailTestServer(t, databaseURL)
defer s.closeDatabases()
ben, err := s.userManager.User("ben")
require.Nil(t, err)
// Link only
rr := requestAdminSocket(t, s, "POST", "/v1/users/password/reset", `{"username":"ben"}`)
require.Equal(t, 200, rr.Code)
var resp apiUsersPasswordResetResponse
require.Nil(t, json.NewDecoder(rr.Body).Decode(&resp))
require.Equal(t, "", resp.Email)
token := tokenFromLink(t, resp.Link, "https://ntfy.example.com/account/password/reset/")
link, err := s.userManager.MagicLinkByToken(token)
require.Nil(t, err)
require.Equal(t, ben.ID, link.UserID)
require.Empty(t, mailer.resetLinks)
// Send email: no primary email yet
rr = requestAdminSocket(t, s, "POST", "/v1/users/password/reset", `{"username":"ben","send_email":true}`)
require.Equal(t, 400, rr.Code)
require.Equal(t, 40067, toHTTPError(t, rr.Body.String()).Code)
// Send email to the primary email
require.Nil(t, s.userManager.AddEmail(ben.ID, "ben@example.com"))
require.Nil(t, s.userManager.SetPrimaryEmail(ben.ID, "ben@example.com"))
rr = requestAdminSocket(t, s, "POST", "/v1/users/password/reset", `{"username":"ben","send_email":true}`)
require.Equal(t, 200, rr.Code)
resp = apiUsersPasswordResetResponse{}
require.Nil(t, json.NewDecoder(rr.Body).Decode(&resp))
require.Equal(t, "ben@example.com", resp.Email)
require.Equal(t, resp.Link, mailer.resetLinks["ben@example.com"])
// Unknown user
rr = requestAdminSocket(t, s, "POST", "/v1/users/password/reset", `{"username":"nobody"}`)
require.Equal(t, 40031, toHTTPError(t, rr.Body.String()).Code)
})
}
func TestPasswordReset_Failures(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
conf := newTestConfigWithAuthFile(t, databaseURL)
conf.AuthUsers = []*user.User{
{Name: "philuser", Hash: "$2a$10$U4WSIYY6evyGmZaraavM2e2JeVG6EMGUKN1uUwufUeeRd4Jpg6cGC", Role: user.RoleUser}, // philuser:philpass
}
s := newTestServer(t, conf)
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
auth := map[string]string{"Authorization": util.BasicAuth("phil", "phil")}
// No mailer configured
rr := request(t, s, "POST", "/v1/users/password/reset", `{"username":"ben","send_email":true}`, auth)
require.Equal(t, 400, rr.Code)
// Provisioned user, admin over HTTP
rr = request(t, s, "POST", "/v1/users/password/reset", `{"username":"philuser"}`, auth)
require.Equal(t, 40905, toHTTPError(t, rr.Body.String()).Code)
rr = request(t, s, "POST", "/v1/users/password/reset", `{"username":"phil"}`, auth)
require.Equal(t, 403, rr.Code)
// No base URL
s.config.BaseURL = ""
rr = request(t, s, "POST", "/v1/users/password/reset", `{"username":"ben"}`, auth)
require.Equal(t, 40068, toHTTPError(t, rr.Body.String()).Code)
})
}
func TestTiers_AddListUpdateRemove(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
auth := map[string]string{"Authorization": util.BasicAuth("phil", "phil")}
// Add
rr := request(t, s, "POST", "/v1/tiers", `{"code":"pro","name":"Pro","messages_limit":1000,"messages_expiry_duration":86400,"attachment_file_size_limit":1048576}`, auth)
require.Equal(t, 200, rr.Code)
tier, err := s.userManager.Tier("pro")
require.Nil(t, err)
require.Equal(t, int64(1000), tier.MessageLimit)
require.Equal(t, 24*time.Hour, tier.MessageExpiryDuration)
require.Nil(t, s.userManager.ChangeTier("ben", "pro"))
// List
rr = request(t, s, "GET", "/v1/tiers", "", auth)
require.Equal(t, 200, rr.Code)
var tiers []*apiAdminTier
require.Nil(t, json.NewDecoder(rr.Body).Decode(&tiers))
require.Len(t, tiers, 1)
require.Equal(t, tier.ID, tiers[0].ID)
require.Equal(t, "Pro", tiers[0].Name)
require.Equal(t, int64(86400), tiers[0].MessagesExpiryDuration)
require.Equal(t, int64(1048576), tiers[0].AttachmentFileSizeLimit)
require.Equal(t, int64(1), tiers[0].Users)
// Update
rr = request(t, s, "PUT", "/v1/tiers", `{"code":"pro","name":"Pro Plus","messages_limit":2000}`, auth)
require.Equal(t, 200, rr.Code)
tier, err = s.userManager.Tier("pro")
require.Nil(t, err)
require.Equal(t, "Pro Plus", tier.Name)
require.Equal(t, int64(2000), tier.MessageLimit)
// Remove: refused while in use, works after moving the user
rr = request(t, s, "DELETE", "/v1/tiers", `{"code":"pro"}`, auth)
require.Equal(t, 409, rr.Code)
require.Equal(t, 40910, toHTTPError(t, rr.Body.String()).Code)
require.Nil(t, s.userManager.ResetTier("ben"))
rr = request(t, s, "DELETE", "/v1/tiers", `{"code":"pro"}`, auth)
require.Equal(t, 200, rr.Code)
_, err = s.userManager.Tier("pro")
require.ErrorIs(t, err, user.ErrTierNotFound)
})
}
func TestTiers_Failures(t *testing.T) {
forEachBackend(t, func(t *testing.T, databaseURL string) {
s := newTestServer(t, newTestConfigWithAuthFile(t, databaseURL))
defer s.closeDatabases()
require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleAdmin, false))
require.Nil(t, s.userManager.AddUser("ben", "ben", user.RoleUser, false))
require.Nil(t, s.userManager.AddTier(&user.Tier{Code: "pro", Name: "Pro"}))
auth := map[string]string{"Authorization": util.BasicAuth("phil", "phil")}
// Duplicate, bad code, missing name
rr := request(t, s, "POST", "/v1/tiers", `{"code":"pro","name":"Pro"}`, auth)
require.Equal(t, 40909, toHTTPError(t, rr.Body.String()).Code)
rr = request(t, s, "POST", "/v1/tiers", `{"code":"not valid","name":"X"}`, auth)
require.Equal(t, 40062, toHTTPError(t, rr.Body.String()).Code)
rr = request(t, s, "POST", "/v1/tiers", `{"code":"x"}`, auth)
require.Equal(t, 40062, toHTTPError(t, rr.Body.String()).Code)
// Update or remove a missing tier
rr = request(t, s, "PUT", "/v1/tiers", `{"code":"missing","name":"Missing"}`, auth)
require.Equal(t, 40030, toHTTPError(t, rr.Body.String()).Code)
rr = request(t, s, "DELETE", "/v1/tiers", `{"code":"missing"}`, auth)
require.Equal(t, 40030, toHTTPError(t, rr.Body.String()).Code)
// Non-admins cannot change tiers; listing them is the billing list (for older web apps), which needs payments
ben := map[string]string{"Authorization": util.BasicAuth("ben", "ben")}
rr = request(t, s, "POST", "/v1/tiers", `{"code":"ben","name":"Ben"}`, ben)
require.Equal(t, 401, rr.Code)
rr = request(t, s, "DELETE", "/v1/tiers", `{"code":"pro"}`, ben)
require.Equal(t, 401, rr.Code)
rr = request(t, s, "GET", "/v1/tiers", "", ben)
require.Equal(t, 404, rr.Code)
rr = request(t, s, "GET", "/v1/tiers", "", nil)
require.Equal(t, 404, rr.Code)
rr = request(t, s, "GET", "/v1/admin/tiers", "", auth) // Gone, never released
require.Equal(t, 404, rr.Code)
})
}
// requestAdminSocket sends a request through the admin socket handler, i.e. with socket semantics
// (no Authorization header, contextAdminSocket set), without a listener.
func requestAdminSocket(t *testing.T, s *Server, method, url, body string) *httptest.ResponseRecorder {
rr := httptest.NewRecorder()
r, err := http.NewRequest(method, url, strings.NewReader(body))
require.Nil(t, err)
s.handleAdminSocket(rr, r)
return rr
}
func mustUserID(t *testing.T, s *Server, username string) string {
u, err := s.userManager.User(username)
require.Nil(t, err)
return u.ID
}
server/server_auth.go
+1 -2
View File
@@ -25,8 +25,7 @@ func (s *Server) maybeAuthenticate(r *http.Request) (*http.Request, *visitor, er
// Read the "Authorization" header value and exit out early if it's not set
ip := extractIPAddress(r, s.config.BehindProxy, s.config.ProxyForwardedHeader, s.config.ProxyTrustedPrefixes)
// Stash the extracted client IP in the request context so downstream code (the abuse ban-feed in
// handleError) can reuse it without re-parsing headers, and so an account-keyed (tier'd) visitor --
// whose shared visitor object has a stale v.ip -- is still attributed to the actual request IP.
// handleError) can reuse it without re-parsing headers.
r = withContext(r, map[contextKey]any{contextVisitorIP: ip})
vip := s.visitor(ip, nil)
if s.userManager == nil {
Loaded 100 of 208 files, more files were not shown because too many files have changed in this diff. Show more