fix(runtime): identify stale leases by boot id and publish them atomically
Comparing the lock directory's mtime with Date.now() - os.uptime() breaks when the wall clock jumps forward after a lease is taken (no RTC, VM resume, NTP or manual correction): a live lease looks older than the boot and is cleared, so a second Runtime can open the same data dir. owner.json now records the kernel boot identity (kern.bootsessionuuid on macOS, /proc/sys/kernel/random/boot_id on Linux, null elsewhere), read once per process. removeStaleLease clears a lease whose boot_id differs from the current one, which still fixes the recycled-PID crash loop of #184 without consulting the clock; when either id is unknown the PID probe decides as before. bootTime, BOOT_SLACK_MS and defaultBootTime are gone; the boot id source is injectable via RuntimeServiceOptions.bootId. The lease is published in one step: owner.json is written into a runtime-service.lock.tmp-<uuid> staging dir that is then renamed onto the lock, so a power cut can no longer leave a lock without an owner. Because rename() replaces an existing empty directory, an existing lock of any kind is reported as taken first. releaseLease also removes the lock with a single rename, so this version never shows an owner-less lock; staging and cleared-lease debris is swept once the lease is held. An owner-less lock can therefore only come from an older version. It is cleared when its mtime predates the kernel-reported boot time (kern.boottime, /proc/stat btime) and otherwise keeps failing as runtime_service_lock_invalid, as before. Refs #184 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a17230dadb
commit
63a80092c2
2 files changed
+460
-134
No files matched your search
@@ -2,9 +2,11 @@ import {
|
||||
chmod,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readdir,
|
||||
readFile,
|
||||
rm,
|
||||
stat,
|
||||
symlink,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
@@ -38,6 +40,38 @@ function supervisor() {
|
||||
};
|
||||
}
|
||||
|
||||
const PREVIOUS_BOOT = "00000000-0000-4000-8000-00000000000a";
|
||||
const CURRENT_BOOT = "00000000-0000-4000-8000-00000000000b";
|
||||
|
||||
/** Seed a lock as an existing owner would leave it; no owner = legacy debris. */
|
||||
async function seedLock(dataDir: string, owner?: object): Promise<string> {
|
||||
const lock = path.join(dataDir, "runtime-service.lock");
|
||||
await mkdir(lock, { mode: 0o700 });
|
||||
if (owner) {
|
||||
await writeFile(path.join(lock, "owner.json"), JSON.stringify(owner), {
|
||||
mode: 0o600,
|
||||
});
|
||||
}
|
||||
await chmod(lock, 0o700);
|
||||
return lock;
|
||||
}
|
||||
|
||||
async function ownerOf(dataDir: string): Promise<Record<string, unknown>> {
|
||||
return JSON.parse(
|
||||
await readFile(
|
||||
path.join(dataDir, "runtime-service.lock", "owner.json"),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** Every lock, staging and cleared-lease entry in the data dir. */
|
||||
async function lockEntries(dataDir: string): Promise<string[]> {
|
||||
return (await readdir(dataDir))
|
||||
.filter((name) => name.startsWith("runtime-service.lock"))
|
||||
.sort();
|
||||
}
|
||||
|
||||
describe("RuntimeService", () => {
|
||||
it("publishes and removes an outer Runtime receipt without secrets", async () => {
|
||||
const dataDir = await directory("runtime-service-");
|
||||
@@ -71,18 +105,7 @@ describe("RuntimeService", () => {
|
||||
|
||||
it("refuses a live owner of the same Runtime home", async () => {
|
||||
const dataDir = await directory("runtime-service-live-");
|
||||
const lock = path.join(dataDir, "runtime-service.lock");
|
||||
await import("node:fs/promises").then(({ mkdir }) =>
|
||||
mkdir(lock, { mode: 0o700 }),
|
||||
);
|
||||
await writeFile(
|
||||
path.join(lock, "owner.json"),
|
||||
JSON.stringify({ pid: 777 }),
|
||||
{
|
||||
mode: 0o600,
|
||||
},
|
||||
);
|
||||
await chmod(lock, 0o700);
|
||||
await seedLock(dataDir, { pid: 777 });
|
||||
const service = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
@@ -94,104 +117,228 @@ describe("RuntimeService", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("reclaims a lease older than the current boot even when a recycled PID answers the liveness probe", async () => {
|
||||
it("reclaims a previous-boot lease even when a recycled PID answers the liveness probe", async () => {
|
||||
// Issue #184: a forced power-off leaves runtime-service.lock behind;
|
||||
// after the reboot the owner's PID is another process, so the old
|
||||
// after the reboot the owner's PID belongs to another process, so a
|
||||
// liveness-only check reports "already running" forever and launchd's
|
||||
// KeepAlive loops the wrapper. A lease from a previous boot can never
|
||||
// have a living owner, regardless of what the PID table says.
|
||||
// KeepAlive loops the wrapper. A lease from another boot can never have
|
||||
// a living owner, whatever the PID table says.
|
||||
const dataDir = await directory("runtime-service-prevboot-");
|
||||
const lock = path.join(dataDir, "runtime-service.lock");
|
||||
await import("node:fs/promises").then(({ mkdir }) =>
|
||||
mkdir(lock, { mode: 0o700 }),
|
||||
);
|
||||
await writeFile(
|
||||
path.join(lock, "owner.json"),
|
||||
JSON.stringify({ pid: 777 }),
|
||||
{
|
||||
mode: 0o600,
|
||||
},
|
||||
);
|
||||
await chmod(lock, 0o700);
|
||||
const leaseTime = Date.now();
|
||||
await seedLock(dataDir, { pid: 777, boot_id: PREVIOUS_BOOT });
|
||||
const service = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
// The recycled PID answers the probe: the liveness check alone
|
||||
// would refuse forever.
|
||||
pid: 4321,
|
||||
processIsAlive: () => true,
|
||||
// This boot began minutes after the lease was taken.
|
||||
bootTime: () => leaseTime + 10 * 60_000,
|
||||
bootId: () => CURRENT_BOOT,
|
||||
});
|
||||
|
||||
await service.start();
|
||||
await service.stop();
|
||||
});
|
||||
|
||||
it("reclaims a lease whose owner.json was never written once the machine has rebooted", async () => {
|
||||
// Issue #184's hard case: a power cut inside the mkdir → privateWrite
|
||||
// window leaves the lock directory with no owner.json at all. That used
|
||||
// to be runtime_service_lock_invalid forever; after a reboot it is
|
||||
// provably from a previous boot and is cleared.
|
||||
const dataDir = await directory("runtime-service-prevboot- corrupt-");
|
||||
const lock = path.join(dataDir, "runtime-service.lock");
|
||||
await import("node:fs/promises").then(({ mkdir }) =>
|
||||
mkdir(lock, { mode: 0o700 }),
|
||||
);
|
||||
await chmod(lock, 0o700);
|
||||
const leaseTime = Date.now();
|
||||
const service = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
processIsAlive: () => true,
|
||||
bootTime: () => leaseTime + 10 * 60_000,
|
||||
expect(await ownerOf(dataDir)).toMatchObject({
|
||||
pid: 4321,
|
||||
boot_id: CURRENT_BOOT,
|
||||
});
|
||||
|
||||
await service.start();
|
||||
await service.stop();
|
||||
expect(await lockEntries(dataDir)).toEqual([]);
|
||||
});
|
||||
|
||||
it("still refuses a corrupt lease from the current boot", async () => {
|
||||
// Same-boot corruption stays refused: the boot-time shortcut only
|
||||
// applies when the lease provably predates the boot.
|
||||
const dataDir = await directory("runtime-service-corrupt-sameboot-");
|
||||
const lock = path.join(dataDir, "runtime-service.lock");
|
||||
await import("node:fs/promises").then(({ mkdir }) =>
|
||||
mkdir(lock, { mode: 0o700 }),
|
||||
);
|
||||
await chmod(lock, 0o700);
|
||||
it("refuses a live owner from the current boot", async () => {
|
||||
const dataDir = await directory("runtime-service-sameboot-live-");
|
||||
await seedLock(dataDir, { pid: 777, boot_id: CURRENT_BOOT });
|
||||
const service = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
bootTime: () => Date.now() - 60 * 60_000, // booted an hour ago
|
||||
pid: 4321,
|
||||
processIsAlive: (pid) => pid === 777,
|
||||
bootId: () => CURRENT_BOOT,
|
||||
});
|
||||
|
||||
await expect(service.start()).rejects.toThrow(
|
||||
"runtime_service_lock_invalid",
|
||||
"runtime_service_already_running",
|
||||
);
|
||||
expect(await ownerOf(dataDir)).toMatchObject({ pid: 777 });
|
||||
});
|
||||
|
||||
it("reclaims a dead owner from the current boot", async () => {
|
||||
const dataDir = await directory("runtime-service-sameboot-dead-");
|
||||
await seedLock(dataDir, { pid: 777, boot_id: CURRENT_BOOT });
|
||||
const service = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
pid: 4321,
|
||||
processIsAlive: () => false,
|
||||
bootId: () => CURRENT_BOOT,
|
||||
});
|
||||
|
||||
await service.start();
|
||||
expect(await ownerOf(dataDir)).toMatchObject({ pid: 4321 });
|
||||
await service.stop();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["the lease predates boot ids", { pid: 777 }, CURRENT_BOOT],
|
||||
["the lease has an empty boot id", { pid: 777, boot_id: "" }, CURRENT_BOOT],
|
||||
[
|
||||
"the current boot id is unknown",
|
||||
{ pid: 777, boot_id: PREVIOUS_BOOT },
|
||||
null,
|
||||
],
|
||||
])("falls back to the PID probe when %s", async (_case, owner, current) => {
|
||||
const dataDir = await directory("runtime-service-unknown-boot-");
|
||||
await seedLock(dataDir, owner);
|
||||
const live = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
pid: 4321,
|
||||
processIsAlive: (pid) => pid === 777,
|
||||
bootId: () => current,
|
||||
});
|
||||
await expect(live.start()).rejects.toThrow(
|
||||
"runtime_service_already_running",
|
||||
);
|
||||
expect(await ownerOf(dataDir)).toMatchObject({ pid: 777 });
|
||||
|
||||
const dead = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
pid: 4321,
|
||||
processIsAlive: () => false,
|
||||
bootId: () => current,
|
||||
});
|
||||
await dead.start();
|
||||
expect(await ownerOf(dataDir)).toMatchObject({ pid: 4321 });
|
||||
await dead.stop();
|
||||
});
|
||||
|
||||
it("records the platform boot id in owner.json", async () => {
|
||||
const dataDir = await directory("runtime-service-boot-id-");
|
||||
const service = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
});
|
||||
|
||||
await service.start();
|
||||
const owner = await ownerOf(dataDir);
|
||||
if (process.platform === "darwin" || process.platform === "linux") {
|
||||
expect(owner.boot_id).toMatch(/^[0-9A-Za-z-]{8,64}$/);
|
||||
} else {
|
||||
expect(owner.boot_id).toBeNull();
|
||||
}
|
||||
expect(owner.pid).toBe(process.pid);
|
||||
await service.stop();
|
||||
});
|
||||
|
||||
it("leaves no lock behind when publishing fails before the rename", async () => {
|
||||
// The lock only appears through a rename of a fully written staging
|
||||
// directory, so a failure (or power cut) while staging cannot leave a
|
||||
// lock without owner.json.
|
||||
const dataDir = await directory("runtime-service-publish-failure-");
|
||||
const broken = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
bootId: () => {
|
||||
throw new Error("simulated crash while staging");
|
||||
},
|
||||
});
|
||||
|
||||
await expect(broken.start()).rejects.toThrow(
|
||||
"runtime_service_lock_unavailable",
|
||||
);
|
||||
expect(await lockEntries(dataDir)).toEqual([]);
|
||||
|
||||
const next = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
pid: 4321,
|
||||
bootId: () => CURRENT_BOOT,
|
||||
});
|
||||
await next.start();
|
||||
expect(await ownerOf(dataDir)).toMatchObject({ pid: 4321 });
|
||||
await next.stop();
|
||||
});
|
||||
|
||||
it("clears an owner-less legacy lock created before the current boot", async () => {
|
||||
// Older versions created the lock with mkdir and wrote owner.json
|
||||
// afterwards; a power cut in between left an empty lock that used to
|
||||
// fail as runtime_service_lock_invalid forever.
|
||||
const dataDir = await directory("runtime-service-legacy-prevboot-");
|
||||
const lock = await seedLock(dataDir);
|
||||
const createdAt = (await stat(lock)).mtimeMs;
|
||||
const service = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
pid: 4321,
|
||||
bootId: () => CURRENT_BOOT,
|
||||
legacyBootTime: () => createdAt + 60_000,
|
||||
});
|
||||
|
||||
await service.start();
|
||||
expect(await ownerOf(dataDir)).toMatchObject({ pid: 4321 });
|
||||
await service.stop();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["it may be from the current boot", (createdAt: number) => createdAt - 1],
|
||||
["the boot time is unknown", () => null],
|
||||
])(
|
||||
"keeps refusing an owner-less legacy lock when %s",
|
||||
async (_case, boot) => {
|
||||
const dataDir = await directory("runtime-service-legacy-sameboot-");
|
||||
const lock = await seedLock(dataDir);
|
||||
const createdAt = (await stat(lock)).mtimeMs;
|
||||
const service = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
bootId: () => CURRENT_BOOT,
|
||||
legacyBootTime: () => boot(createdAt),
|
||||
});
|
||||
|
||||
await expect(service.start()).rejects.toThrow(
|
||||
"runtime_service_lock_invalid",
|
||||
);
|
||||
// rename() may replace an empty directory; the publish step must not.
|
||||
expect(await readdir(lock)).toEqual([]);
|
||||
expect(await lockEntries(dataDir)).toEqual(["runtime-service.lock"]);
|
||||
},
|
||||
);
|
||||
|
||||
it("removes staging and cleared-lease debris after acquiring", async () => {
|
||||
const dataDir = await directory("runtime-service-debris-");
|
||||
const staging = path.join(dataDir, "runtime-service.lock.tmp-crashed");
|
||||
await mkdir(staging, { mode: 0o700 });
|
||||
await writeFile(
|
||||
path.join(staging, "owner.json"),
|
||||
JSON.stringify({ pid: 9 }),
|
||||
);
|
||||
await mkdir(path.join(dataDir, "runtime-service.lock.stale.crashed"), {
|
||||
mode: 0o700,
|
||||
});
|
||||
const service = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
pid: 4321,
|
||||
bootId: () => CURRENT_BOOT,
|
||||
});
|
||||
|
||||
await service.start();
|
||||
expect(await lockEntries(dataDir)).toEqual(["runtime-service.lock"]);
|
||||
expect(await ownerOf(dataDir)).toMatchObject({ pid: 4321 });
|
||||
await service.stop();
|
||||
expect(await lockEntries(dataDir)).toEqual([]);
|
||||
});
|
||||
|
||||
it("reclaims a stale validated lease", async () => {
|
||||
const dataDir = await directory("runtime-service-stale-");
|
||||
const lock = path.join(dataDir, "runtime-service.lock");
|
||||
await import("node:fs/promises").then(({ mkdir }) =>
|
||||
mkdir(lock, { mode: 0o700 }),
|
||||
);
|
||||
await writeFile(
|
||||
path.join(lock, "owner.json"),
|
||||
JSON.stringify({ pid: 777 }),
|
||||
{
|
||||
mode: 0o600,
|
||||
},
|
||||
);
|
||||
await chmod(lock, 0o700);
|
||||
await seedLock(dataDir, { pid: 777 });
|
||||
const service = new RuntimeService({
|
||||
dataDir,
|
||||
createSupervisor: supervisor,
|
||||
pid: 4321,
|
||||
processIsAlive: () => false,
|
||||
});
|
||||
|
||||
await service.start();
|
||||
expect(await ownerOf(dataDir)).toMatchObject({ pid: 4321 });
|
||||
await service.stop();
|
||||
});
|
||||
|
||||
@@ -463,7 +610,7 @@ describe("RuntimeService", () => {
|
||||
const real = path.join(parent, "real");
|
||||
const link = path.join(parent, "data");
|
||||
await mkdir(real, { mode: 0o755 });
|
||||
await import("node:fs/promises").then(({ symlink }) => symlink(real, link));
|
||||
await symlink(real, link);
|
||||
const service = new RuntimeService({
|
||||
dataDir: link,
|
||||
createSupervisor: () => supervisor(),
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { readFileSync, type Stats } from "node:fs";
|
||||
import {
|
||||
chmod,
|
||||
lstat,
|
||||
mkdir,
|
||||
readdir,
|
||||
readFile,
|
||||
rename,
|
||||
rm,
|
||||
rmdir,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { processIsAlive } from "./core-supervisor.js";
|
||||
@@ -36,10 +37,14 @@ export interface RuntimeServiceOptions {
|
||||
): ManagedRuntime | Promise<ManagedRuntime>;
|
||||
pid?: number;
|
||||
processIsAlive?: (pid: number) => boolean;
|
||||
/** Wall-clock epoch (ms) of the current boot. The default is derived from
|
||||
* `os.uptime()`; injectable so tests can put a lease before or after the
|
||||
* boot without moving the machine's clock. */
|
||||
bootTime?: () => number;
|
||||
/** Identity of the current boot (`kern.bootsessionuuid` on macOS,
|
||||
* `/proc/sys/kernel/random/boot_id` on Linux, `null` when unknown).
|
||||
* Injectable so tests can simulate a reboot without one. */
|
||||
bootId?: () => string | null;
|
||||
/** Wall-clock epoch (ms) the kernel reports for the current boot, or `null`
|
||||
* when unknown. Consulted only for a legacy lock directory without
|
||||
* owner.json; see removeUnownedLease. Injectable for tests. */
|
||||
legacyBootTime?: () => number | null;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
@@ -50,9 +55,6 @@ interface LeaseIdentity {
|
||||
|
||||
/** Publishes one outer Runtime identity around the Core+sidecar supervisor. */
|
||||
export class RuntimeService {
|
||||
/** See removeStaleLease: absorbs a forward clock step since boot. */
|
||||
private static readonly BOOT_SLACK_MS = 2 * 60_000;
|
||||
|
||||
private readonly options: RuntimeServiceOptions;
|
||||
private lease: LeaseIdentity | null = null;
|
||||
private supervisor: ManagedRuntime | null = null;
|
||||
@@ -194,29 +196,81 @@ export class RuntimeService {
|
||||
const lock = this.lockPath();
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
try {
|
||||
await mkdir(lock, { mode: 0o700 });
|
||||
const identity = await lstat(lock);
|
||||
this.lease = { dev: identity.dev, ino: identity.ino };
|
||||
try {
|
||||
await privateWrite(
|
||||
path.join(lock, "owner.json"),
|
||||
JSON.stringify({
|
||||
pid: this.pid(),
|
||||
created_at: new Date().toISOString(),
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
await this.releaseLease();
|
||||
throw error;
|
||||
}
|
||||
return;
|
||||
await this.publishLease(lock);
|
||||
break;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "EEXIST" || attempt > 0) {
|
||||
if (!leaseTaken(error) || attempt > 0) {
|
||||
throw new Error("runtime_service_lock_unavailable", { cause: error });
|
||||
}
|
||||
await this.removeStaleLease(lock);
|
||||
}
|
||||
}
|
||||
await this.sweepLeaseDebris();
|
||||
}
|
||||
|
||||
/** Publish the lease in one step: the lock directory becomes visible with
|
||||
* owner.json (pid, created_at, boot_id) already inside it, so a power cut
|
||||
* can leave a staging directory behind but never a lock without an owner.
|
||||
* The old mkdir -> write sequence had exactly that window (issue #184). */
|
||||
private async publishLease(lock: string): Promise<void> {
|
||||
const staging = `${lock}${STAGING_INFIX}${randomUUID()}`;
|
||||
await mkdir(staging, { mode: 0o700 });
|
||||
try {
|
||||
await privateWrite(
|
||||
path.join(staging, "owner.json"),
|
||||
JSON.stringify({
|
||||
pid: this.pid(),
|
||||
created_at: new Date().toISOString(),
|
||||
boot_id: this.currentBootId(),
|
||||
}),
|
||||
);
|
||||
// POSIX rename() silently REPLACES an existing *empty* directory, so on
|
||||
// its own it is not an exclusive create. A non-empty lock (every lease
|
||||
// this version publishes) makes it fail with EEXIST/ENOTEMPTY. An empty
|
||||
// lock can only come from an older version's two-step mkdir -> write:
|
||||
// either power-cut debris, or a live acquire in flight. The latter must
|
||||
// not be overwritten, and the former has to go through
|
||||
// removeStaleLease's legacy rule rather than be replaced blindly, so an
|
||||
// existing lock of any kind is reported as taken here. The residual race
|
||||
// (an old binary mkdir-ing between this check and the rename) needs an
|
||||
// old and a new Runtime starting on the same home within microseconds;
|
||||
// this version never creates an empty lock, since releaseLease also
|
||||
// removes it with a single rename.
|
||||
if (await pathExists(lock)) throw leaseTakenError();
|
||||
await rename(staging, lock);
|
||||
} catch (error) {
|
||||
await rm(staging, { recursive: true, force: true }).catch(
|
||||
() => undefined,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
// rename() keeps the inode, so this is the published directory's identity;
|
||||
// releaseLease compares it before touching the lock.
|
||||
const identity = await lstat(lock);
|
||||
this.lease = { dev: identity.dev, ino: identity.ino };
|
||||
}
|
||||
|
||||
/** Remove staging and cleared-lease directories left behind by a crash.
|
||||
* Runs only while this process holds the lease, so a staging directory is
|
||||
* either debris or belongs to a competitor that has already lost (its
|
||||
* rename cannot succeed over our non-empty lock). Best-effort. */
|
||||
private async sweepLeaseDebris(): Promise<void> {
|
||||
const base = path.basename(this.lockPath());
|
||||
const names = await readdir(this.options.dataDir).catch((): string[] => []);
|
||||
await Promise.all(
|
||||
names
|
||||
.filter(
|
||||
(name) =>
|
||||
name.startsWith(`${base}${STAGING_INFIX}`) ||
|
||||
name.startsWith(`${base}${STALE_INFIX}`),
|
||||
)
|
||||
.map((name) =>
|
||||
rm(path.join(this.options.dataDir, name), {
|
||||
recursive: true,
|
||||
force: true,
|
||||
}).catch(() => undefined),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private async removeStaleLease(lock: string): Promise<void> {
|
||||
@@ -231,33 +285,47 @@ export class RuntimeService {
|
||||
) {
|
||||
throw new Error("runtime_service_lock_unsafe");
|
||||
}
|
||||
// A lease created before the current boot cannot have a living owner:
|
||||
// every process from that boot is gone, and a PID that answers the
|
||||
// liveness probe now is a different process that merely reused the
|
||||
// number. Before this check, a forced power-off (issue #184) left
|
||||
// runtime-service.lock behind, the reboot recycled the owner's PID, and
|
||||
// launchd's KeepAlive restarted the wrapper into
|
||||
// runtime_service_already_running without end.
|
||||
//
|
||||
// The slack absorbs a forward clock step since boot: an NTP correction
|
||||
// would otherwise make a lease taken moments after boot look older than
|
||||
// the boot itself. With the slack, only a jump larger than two minutes
|
||||
// within that window could mistake a live lease for a stale one.
|
||||
const bootTime = this.options.bootTime ?? defaultBootTime;
|
||||
if (info.mtimeMs < bootTime() - RuntimeService.BOOT_SLACK_MS) {
|
||||
await this.clearLeaseDir(lock);
|
||||
return;
|
||||
let raw: string;
|
||||
try {
|
||||
raw = await readFile(path.join(lock, "owner.json"), "utf8");
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
await this.removeUnownedLease(lock, info);
|
||||
return;
|
||||
}
|
||||
throw new Error("runtime_service_lock_invalid", { cause: error });
|
||||
}
|
||||
let owner: unknown;
|
||||
try {
|
||||
owner = JSON.parse(await readFile(path.join(lock, "owner.json"), "utf8"));
|
||||
owner = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new Error("runtime_service_lock_invalid");
|
||||
}
|
||||
const pid =
|
||||
owner && typeof owner === "object"
|
||||
? (owner as { pid?: unknown }).pid
|
||||
: undefined;
|
||||
if (!owner || typeof owner !== "object" || Array.isArray(owner)) {
|
||||
throw new Error("runtime_service_lock_invalid");
|
||||
}
|
||||
// A lease written during another boot cannot have a living owner: every
|
||||
// process of that boot is gone, and a PID that answers the liveness probe
|
||||
// now merely reused the number. Without this, a forced power-off left the
|
||||
// lock behind, the reboot recycled the owner's PID, and launchd's
|
||||
// KeepAlive restarted the wrapper into runtime_service_already_running
|
||||
// without end (issue #184). The kernel's boot identity is compared, not
|
||||
// the wall clock: the clock can jump forward while a lease is held (no
|
||||
// RTC, VM resume, NTP or manual correction) and make a live lease look
|
||||
// older than the boot. When either side is unknown (a lease from an older
|
||||
// version, an unsupported platform) the PID probe below decides, as before.
|
||||
const ownerBoot = (owner as { boot_id?: unknown }).boot_id;
|
||||
const currentBoot = this.currentBootId();
|
||||
if (
|
||||
typeof ownerBoot === "string" &&
|
||||
ownerBoot !== "" &&
|
||||
currentBoot !== null &&
|
||||
ownerBoot !== currentBoot
|
||||
) {
|
||||
await this.clearLeaseDir(lock);
|
||||
return;
|
||||
}
|
||||
const pid = (owner as { pid?: unknown }).pid;
|
||||
if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 1) {
|
||||
throw new Error("runtime_service_lock_invalid");
|
||||
}
|
||||
@@ -267,10 +335,31 @@ export class RuntimeService {
|
||||
await this.clearLeaseDir(lock);
|
||||
}
|
||||
|
||||
/** A lock directory without owner.json. This version publishes owner.json
|
||||
* together with the lock and releases it with one rename, so such a
|
||||
* directory was written by an older version (mkdir -> write, where a power
|
||||
* cut in between leaves it empty) or by hand. It names no PID and no boot,
|
||||
* so the only usable evidence is "created before the current boot": clear
|
||||
* it then, and otherwise keep the previous refusal
|
||||
* (runtime_service_lock_invalid), including when the boot time is unknown.
|
||||
*
|
||||
* The kernel boot time and the directory mtime are both wall-clock values,
|
||||
* which is why lease ownership itself no longer depends on them. Here a
|
||||
* forward clock jump can only hurt if an older binary is live inside its
|
||||
* mkdir -> write window at that very moment; a backward jump only keeps the
|
||||
* old refusal. */
|
||||
private async removeUnownedLease(lock: string, info: Stats): Promise<void> {
|
||||
const bootTime = (this.options.legacyBootTime ?? currentBootTimeMs)();
|
||||
if (bootTime === null || !(info.mtimeMs < bootTime)) {
|
||||
throw new Error("runtime_service_lock_invalid");
|
||||
}
|
||||
await this.clearLeaseDir(lock);
|
||||
}
|
||||
|
||||
private async clearLeaseDir(lock: string): Promise<void> {
|
||||
const stale = `${lock}.stale.${randomUUID()}`;
|
||||
const stale = `${lock}${STALE_INFIX}${randomUUID()}`;
|
||||
await rename(lock, stale);
|
||||
await rm(stale, { recursive: true });
|
||||
await rm(stale, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
private async releaseLease(): Promise<void> {
|
||||
@@ -281,8 +370,19 @@ export class RuntimeService {
|
||||
const current = await lstat(lock).catch(() => null);
|
||||
if (!current || current.dev !== lease.dev || current.ino !== lease.ino)
|
||||
return;
|
||||
await rm(path.join(lock, "owner.json"), { force: true });
|
||||
await rmdir(lock).catch(() => undefined);
|
||||
// Take the lock away with one rename so no observer ever sees it without
|
||||
// owner.json: an owner-less lock is left to mean legacy debris.
|
||||
const released = `${lock}${STALE_INFIX}${randomUUID()}`;
|
||||
try {
|
||||
await rename(lock, released);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
await rm(released, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
|
||||
private currentBootId(): string | null {
|
||||
return (this.options.bootId ?? currentBootId)();
|
||||
}
|
||||
|
||||
private async writeReceipt(
|
||||
@@ -331,10 +431,89 @@ export class RuntimeService {
|
||||
}
|
||||
}
|
||||
|
||||
function defaultBootTime(): number {
|
||||
// Wall-clock epoch of the current boot, as far as the clock admits today.
|
||||
// os.uptime() is boot-based (includes suspend) on every supported platform.
|
||||
return Date.now() - os.uptime() * 1000;
|
||||
const STAGING_INFIX = ".tmp-";
|
||||
const STALE_INFIX = ".stale.";
|
||||
|
||||
function leaseTaken(error: unknown): boolean {
|
||||
const code = (error as NodeJS.ErrnoException | undefined)?.code;
|
||||
return code === "EEXIST" || code === "ENOTEMPTY";
|
||||
}
|
||||
|
||||
function leaseTakenError(): NodeJS.ErrnoException {
|
||||
return Object.assign(new Error("runtime_service_lock_exists"), {
|
||||
code: "EEXIST",
|
||||
});
|
||||
}
|
||||
|
||||
async function pathExists(target: string): Promise<boolean> {
|
||||
try {
|
||||
await lstat(target);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function sysctl(name: string): string {
|
||||
return execFileSync("/usr/sbin/sysctl", ["-n", name], {
|
||||
encoding: "utf8",
|
||||
timeout: 1000,
|
||||
maxBuffer: 4096,
|
||||
killSignal: "SIGKILL",
|
||||
shell: false,
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
env: { PATH: "/usr/bin:/bin:/usr/sbin:/sbin", LANG: "C", LC_ALL: "C" },
|
||||
});
|
||||
}
|
||||
|
||||
let cachedBootId: string | null | undefined;
|
||||
|
||||
/** The kernel's identity for the current boot, cached per process (it cannot
|
||||
* change while the process lives). `null` on unsupported platforms or when
|
||||
* the source cannot be read; the lease logic then falls back to the PID
|
||||
* probe. */
|
||||
function currentBootId(): string | null {
|
||||
if (cachedBootId === undefined) cachedBootId = readBootId();
|
||||
return cachedBootId;
|
||||
}
|
||||
|
||||
function readBootId(): string | null {
|
||||
try {
|
||||
let raw: string;
|
||||
if (process.platform === "darwin") raw = sysctl("kern.bootsessionuuid");
|
||||
else if (process.platform === "linux")
|
||||
raw = readFileSync("/proc/sys/kernel/random/boot_id", "utf8");
|
||||
else return null;
|
||||
const id = raw.trim();
|
||||
return /^[0-9A-Za-z-]{8,64}$/.test(id) ? id : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Wall-clock epoch (ms) the kernel reports for the current boot, or `null`.
|
||||
* Used only for owner-less legacy lock directories. */
|
||||
function currentBootTimeMs(): number | null {
|
||||
try {
|
||||
if (process.platform === "darwin") {
|
||||
const match = /\bsec\s*=\s*(\d+)\s*,\s*usec\s*=\s*(\d+)/.exec(
|
||||
sysctl("kern.boottime"),
|
||||
);
|
||||
return match
|
||||
? Number(match[1]) * 1000 + Math.floor(Number(match[2]) / 1000)
|
||||
: null;
|
||||
}
|
||||
if (process.platform === "linux") {
|
||||
const match = /^btime\s+(\d+)\s*$/m.exec(
|
||||
readFileSync("/proc/stat", "utf8"),
|
||||
);
|
||||
return match ? Number(match[1]) * 1000 : null;
|
||||
}
|
||||
} catch {
|
||||
// Unknown boot time: the caller keeps refusing.
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function privateWrite(target: string, value: string): Promise<void> {
|
||||
|
||||
Reference in new issue
Block a user