vrr/openclaw
2
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-10-03 01:29:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 29

feat: back up to external disks and Cloudflare R2 with storage locations (#161913)

Browse source 
Adds named storage locations as a generic, pluggable capability, with backup as its first consumer.

- Core storage owner (src/storage): storage.locations config, a location marker that binds identity (runtime never creates it, so unplugged disks and different disks at the same path are refused), client-side streaming encryption (scrypt key from a SecretRef passphrase, per-object HKDF keys, AES-256-GCM segments), and a built-in filesystem provider for external disks and mounts.
- Plugin SDK: api.registerStorageProvider plus manifest contracts.storageProviders; providers move opaque bytes only.
- Bundled cloudflare plugin: an r2 provider over the S3 API with conditional writes and bounded multipart uploads; auto-enabled when a location uses provider "r2".
- Backups: backup create --to <location> with verified archives, UTC retention, list/verify/restore --from, Gateway-owned offsite schedules (installed Git schedules unchanged), per-installation namespace claims fenced at publication and deletion, backup record for external jobs, backup.status RPC, Doctor/status hints, and a Systems page Backups section.

No config or state migration; the storage section is new and optional. Proof: live R2 and mounted-disk round trips, namespace takeover trace, and a published 2026.9.7 upgrade cell with an existing Git backup schedule.
This commit is contained in:
Peter Steinberger 2026-10-01 02:05:44 -07:00 • committed by GitHub
parent 6ffa1e433a
commit 37ffe3ce94
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
157 changed files with 9535 additions and 479 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.github/labeler.yml vendored
View file

@ -588,6 +588,11 @@
- "extensions/anthropic/**"
- "docs/providers/anthropic.md"
- "docs/plugins/reference/anthropic.md"
"extensions: cloudflare":
- changed-files:
- any-glob-to-any-file:
- "extensions/cloudflare/**"
- "docs/plugins/cloudflare.md"
"extensions: cloudflare-ai-gateway":
- changed-files:
- any-glob-to-any-file:

3
apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt
View file

@ -1049,6 +1049,9 @@ enum class GatewayMethod(
GatewayStopRequest("gateway.stop.request"),
DiagnosticsHeapSnapshot("diagnostics.heapSnapshot"),
SessionsCatalogImport("sessions.catalog.import"),
BackupStatus("backup.status"),
StorageLocationsList("storage.locations.list"),
StorageLocationsProbe("storage.locations.probe"),
}
enum class GatewayEvent(

175
apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift
View file

@ -2551,6 +2551,133 @@ public struct AuditRunInspectResult: Codable, Sendable {
}
}
public struct BackupRunLocation: Codable, Sendable {
public let name: String
public let provider: String
public let locationid: String
public let key: String
public let namespace: String
public let plaintextbytes: Int
public let storedbytes: Int
public init(
name: String,
provider: String,
locationid: String,
key: String,
namespace: String,
plaintextbytes: Int,
storedbytes: Int)
{
self.name = name
self.provider = provider
self.locationid = locationid
self.key = key
self.namespace = namespace
self.plaintextbytes = plaintextbytes
self.storedbytes = storedbytes
}
private enum CodingKeys: String, CodingKey {
case name
case provider
case locationid = "locationId"
case key
case namespace
case plaintextbytes = "plaintextBytes"
case storedbytes = "storedBytes"
}
}
public struct BackupRunRecord: Codable, Sendable {
public let id: String
public let createdat: Double
public let archivepath: String
public let status: AnyCodable
public let kind: AnyCodable
public let target: String?
public let namespace: String?
public let error: String?
public let pushfailed: Bool?
public let bytes: Int?
public let location: BackupRunLocation?
public let retention: BackupRunRetention?
public init(
id: String,
createdat: Double,
archivepath: String,
status: AnyCodable,
kind: AnyCodable,
target: String? = nil,
namespace: String? = nil,
error: String? = nil,
pushfailed: Bool? = nil,
bytes: Int? = nil,
location: BackupRunLocation? = nil,
retention: BackupRunRetention? = nil)
{
self.id = id
self.createdat = createdat
self.archivepath = archivepath
self.status = status
self.kind = kind
self.target = target
self.namespace = namespace
self.error = error
self.pushfailed = pushfailed
self.bytes = bytes
self.location = location
self.retention = retention
}
private enum CodingKeys: String, CodingKey {
case id
case createdat = "createdAt"
case archivepath = "archivePath"
case status
case kind
case target
case namespace
case error
case pushfailed = "pushFailed"
case bytes
case location
case retention
}
}
public struct BackupRunRetention: Codable, Sendable {
public let kept: Int
public let deleted: Int
public init(
kept: Int,
deleted: Int)
{
self.kept = kept
self.deleted = deleted
}
}
public struct BackupStatusParams: Codable, Sendable {}
public struct BackupStatusResult: Codable, Sendable {
public let targets: [[String: AnyCodable]]
public let schedules: [[String: AnyCodable]]
public let locations: [[String: AnyCodable]]
public init(
targets: [[String: AnyCodable]],
schedules: [[String: AnyCodable]],
locations: [[String: AnyCodable]])
{
self.targets = targets
self.schedules = schedules
self.locations = locations
}
}
public struct BoardCanvasDocumentSource: Codable, Sendable {
public let kind: String
public let docid: String
@ -20651,6 +20778,54 @@ public struct StateVersion: Codable, Sendable {
}
}
public struct StorageLocationsListParams: Codable, Sendable {}
public struct StorageLocationsListResult: Codable, Sendable {
public let locations: [[String: AnyCodable]]
public init(
locations: [[String: AnyCodable]])
{
self.locations = locations
}
}
public struct StorageLocationsProbeParams: Codable, Sendable {
public let name: String
public init(
name: String)
{
self.name = name
}
}
public struct StorageLocationsProbeResult: Codable, Sendable {
public let state: AnyCodable
public let freebytes: Double?
public let totalbytes: Double?
public let message: String?
public init(
state: AnyCodable,
freebytes: Double? = nil,
totalbytes: Double? = nil,
message: String? = nil)
{
self.state = state
self.freebytes = freebytes
self.totalbytes = totalbytes
self.message = message
}
private enum CodingKeys: String, CodingKey {
case state
case freebytes = "freeBytes"
case totalbytes = "totalBytes"
case message
}
}
public struct SystemAgentApprovalPresentation: Codable, Sendable {
public let kind: String
public let title: String

2
config/knip.config.ts
View file

@ -134,6 +134,8 @@ const repositoryScriptEntries = [
"scripts/e2e/lib/upgrade-survivor/abandoned-update.mjs!",
// backup-rollback.sh invokes capture and verification through this CLI.
"scripts/e2e/lib/upgrade-survivor/backup-rollback.mjs!",
// run.sh invokes the backup schedule upgrade scenario through this CLI.
"scripts/e2e/lib/upgrade-survivor/backup-schedule.mjs!",
"scripts/e2e/lib/upgrade-survivor/channel-owner-policy.mjs!",
"scripts/e2e/lib/upgrade-survivor/config-parking.mjs!",
"scripts/e2e/lib/upgrade-survivor/custom-plugin-siblings.mjs!",

4
docs/.generated/config-baseline.counts.json
View file

@ -1,5 +1,5 @@
{
"core": 2475,
"core": 2486,
"channel": 3786,
"plugin": 4447
"plugin": 4468
}

6
docs/.generated/config-baseline.sha256
View file

@ -1,4 +1,4 @@
d2a8331d19429b051b73805808127fd5d4f2f38b9d02fcb6b8b4d3c8ee9f4e46 config-baseline.json
f6bd94b6be80ed8eec2bbf2a8d0e6a0911dc3c9821fddc50af3ae3ebe1f30b53 config-baseline.core.json
2b8c3b36bdf69f80b77b6a84e61120798da1d0067be91500a18599cd683cfd42 config-baseline.json
e7081e85f3e502fe03f1c0d8c925d14409f7620b7cd507bc5d1e55977570cb08 config-baseline.core.json
94eda411a4334bba78ecb01c8dd6b9c0ddab9a6b61c7a5cbe514cebc6661c753 config-baseline.channel.json
8b05fd2909c392c286b5dd1bff41f19049b5ad36b43c7dc1b88e8021ce7c658e config-baseline.plugin.json
1484b679183ce067794b83b78f3cb21580fa16b0ed53d69957f6761d42fc1534 config-baseline.plugin.json

28
docs/.i18n/glossary.zh-CN.json
View file

@ -807,10 +807,26 @@
"source": "Kubernetes",
"target": "Kubernetes"
},
{
"source": "Cloudflare",
"target": "Cloudflare"
},
{
"source": "Cloudflare Containers",
"target": "Cloudflare Containers"
},
{
"source": "Cloudflare plugin",
"target": "Cloudflare 插件"
},
{
"source": "Cloudflare plugin reference",
"target": "Cloudflare 插件参考"
},
{
"source": "cloudflare",
"target": "cloudflare"
},
{
"source": "Fly.io",
"target": "Fly.io"
@ -4351,6 +4367,18 @@
"source": "Storage changes and release preflight",
"target": "存储变更与发布预检"
},
{
"source": "storage",
"target": "storage"
},
{
"source": "Storage locations",
"target": "存储位置"
},
{
"source": "Storage CLI",
"target": "存储 CLI"
},
{
"source": "Agent schema history",
"target": "Agent 数据库 schema 历史"

250
docs/cli/backup.md
View file

@ -1,9 +1,11 @@
---
summary: "CLI reference for `openclaw backup` (archives, SQLite snapshots, and Git history)"
doc-schema-version: 1
summary: "CLI reference for `openclaw backup` (local and offsite archives, SQLite snapshots, Git history, and schedules)"
read_when:
- You want a first-class backup archive for local OpenClaw state
- You need a compact, verified snapshot of one OpenClaw SQLite database
- You want scheduled, versioned database backups in an operator-owned Git repository
- You want encrypted offsite archives, retention, or external backup status
- You want to preview which paths would be included before reset or uninstall
- You want to restore from a `.tar.gz` archive previously created by `openclaw backup`
title: "Backup"
@ -11,7 +13,10 @@ title: "Backup"
# `openclaw backup`
Create a local backup archive for OpenClaw state, config, auth profiles, channel/provider credentials, sessions, and optionally workspaces.
Create backup archives for OpenClaw state, config, auth profiles, channel/provider
credentials, sessions, and optionally workspaces. Save them locally or upload to
a named [storage location](/concepts/storage-locations). SQLite snapshots and Git
history provide database-focused alternatives.
```bash
openclaw backup create
@ -20,6 +25,10 @@ openclaw backup create --dry-run --json
openclaw backup create --verify
openclaw backup create --no-include-workspace
openclaw backup create --only-config
openclaw backup create --to offsite --keep-daily 7 --keep-weekly 4 --keep-monthly 12
openclaw backup list --from offsite
openclaw backup verify --from offsite latest
openclaw backup restore --from offsite latest --target ./restored-openclaw
openclaw backup verify ./2026-03-09T08-00-00.000+08-00-openclaw-backup.tar.gz
openclaw backup restore ./2026-03-09T08-00-00.000+08-00-openclaw-backup.tar.gz --target ./restored-openclaw
openclaw backup sqlite create --global --repository ~/Backups/openclaw-sqlite
@ -34,16 +43,20 @@ openclaw backup git log --repository ~/Backups/openclaw-git
openclaw backup git verify --repository ~/Backups/openclaw-git --global
openclaw backup git restore --repository ~/Backups/openclaw-git --agent main --target ./restored/agent.sqlite
openclaw backup enable --repository ~/Backups/openclaw-git --every 24h --push
openclaw backup enable --to offsite --every 24h --keep-daily 7
openclaw backup disable --offsite
openclaw backup disable
openclaw backup record --status ok --target host-restic --bytes 1048576
```
Archive `create`, `verify`, and `restore`, plus SQLite `create`, `list`, `verify`, and
`restore`, accept `--json` for one machine-readable result on stdout.
Archive `create`, `list`, `verify`, and `restore`, external `record`, plus SQLite
`create`, `list`, `verify`, and `restore`, accept `--json` for one machine-readable
result on stdout.
## Notes
- The archive embeds a schema-version-1 `manifest.json` with the resolved source paths and archive layout. Additive ownership metadata records configured agent ids and roots, including agent roots already covered by another asset; existing archive layout and older archives remain supported. New archives also record the canonical SQLite snapshots captured at creation; standalone verification rejects missing or mismatched inventory entries. Legacy archives without this inventory remain readable, but verification reports `sqliteInventoryVerified: false` because complete database coverage cannot be established. An empty inventory means no canonical databases were captured (for example, a config-only export), not a full database recovery point.
- Default output is a timestamped `.tar.gz` archive in the current working directory. Timestamped filenames use your machine's local timezone and include the UTC offset. If the current working directory is inside a backed-up source tree, OpenClaw falls back to your home directory for the default archive location.
- Without `--to`, default output is a timestamped `.tar.gz` archive in the current working directory. Local timestamped filenames use your machine's local timezone and include the UTC offset. If the current working directory is inside a backed-up source tree, OpenClaw falls back to your home directory for the default archive location. With `--to`, the default archive is temporary; pass `--output` as well to retain a local copy.
- Existing archive files are never overwritten. Output paths inside the source state/workspace trees are rejected to avoid self-inclusion.
- `openclaw backup verify <archive>` checks that the archive contains exactly one root manifest, rejects traversal-style archive paths and unsafe symbolic links, confirms every manifest-declared payload exists, and validates the root SQLite snapshot and agent snapshots listed in the manifest or captured durable registry. It rejects sidecars for those snapshots and checks their integrity and database roles, including each agent's identity. Other files, including plugin snapshots already validated during creation, remain opaque during verification and restore. `openclaw backup create --verify` runs that validation immediately after writing the archive.
- Full archives include the active config and its required `$include` files, including dependencies outside the state directory. They preserve authored bytes, comments, and environment placeholders; resolved secrets are not written into the config copy. These additional files may contain sensitive data, so protect the archive accordingly.
@ -59,6 +72,123 @@ and its `sqliteSnapshots` inventory, then run `openclaw backup verify <archive>`
A path reported as `covered by` another asset is included through that parent;
it has not been excluded from the archive.
## Offsite archives
Configure a named [storage location](/concepts/storage-locations), initialize it,
and test access before the first backup:
```bash
openclaw storage init offsite
openclaw storage test offsite
openclaw backup create --to offsite
```
The built-in `filesystem` provider supports an existing disk or mounted directory.
The [Cloudflare plugin](/plugins/cloudflare) provides R2 object storage. Storage
configuration owns encryption and credentials; backup commands use the configured
location without provider-specific flags. See [Storage CLI](/cli/storage).
`create --to` opens and checks the location before archiving, creates the archive
in managed scratch, verifies its manifest and payload, uploads it, and confirms
the stored size. Verification is always enabled for offsite creation, even
without `--verify`. The temporary local archive is removed afterward. To keep a
local copy as well, add `--output <path>`; that copy is an ordinary plaintext
`.tar.gz`, even when storage encryption is enabled.
An uninitialized location fails before archive creation and records a failed
attempt with the next step. Reconnect a missing disk or check the bucket and prefix,
or run `openclaw storage
init <name>` only when the destination is new. Backups never initialize storage
implicitly. Keep the encryption passphrase and root location marker available
for recovery.
| Option | Meaning |
| ------------------------ | ------------------------------------------------------------------------------------- |
| `--to <location>` | Upload the verified archive to a configured, initialized location. |
| `--namespace <name>` | Backup namespace; defaults to the sanitized hostname. |
| `--claim-namespace` | Deliberately replace the namespace ownership claim with this installation's identity. |
| `--output <path>` | Also retain a local archive at a path or in a destination directory. |
| `--no-include-workspace` | Omit workspace files while retaining state, config, credentials, and agent databases. |
| `--only-config` | Archive only the active config file; storage configuration must still be readable. |
| `--keep-daily <n>` | Retain the newest backup in each of the newest `n` nonempty UTC days. |
| `--keep-weekly <n>` | Retain the newest backup in each of the newest `n` nonempty UTC weeks. |
| `--keep-monthly <n>` | Retain the newest backup in each of the newest `n` nonempty UTC months. |
Namespaces contain 1–128 letters, digits, dots, underscores, or hyphens and
cannot be `.` or `..`. Objects live under `backups/<namespace>/` with keys such
as `20260930T120000Z-a1b2c3d4.tar.gz`: a UTC timestamp plus eight random hexadecimal
characters. The filename does not change when storage encryption is enabled.
Choose a stable explicit namespace for a host that may be renamed, and use that
same namespace when listing, verifying, or restoring from another host.
The first upload creates `backups/<namespace>/owner.json` with the installation's
durable Gateway device ID, hostname, and claim time. The claim uses the location's
encryption settings. OpenClaw checks that the claim matches this installation
before archiving, at archive publication, and before each retention deletion.
An existing claim with a different device ID refuses the run before archiving
and records a failed attempt naming the owner. Identical
hostnames do not grant shared ownership.
Use a different `--namespace` for a separate installation. To deliberately take
over a stopped or retired installation's namespace, such as after moving to new
hardware, pass `--claim-namespace` with `--to`. This also replaces a damaged ownership claim:
```bash
openclaw backup create --to offsite --namespace gateway --claim-namespace
```
The displaced installation is rejected at its next publication or deletion.
Object stores cannot make an object's write conditional on a separate ownership
claim, so a residual provider round-trip window remains between the final check
and the effect. Stop the old installation before taking over; use a separate
namespace for installations that run concurrently.
A restored installation retains its device identity and can continue using its
namespace. A cloned copy running at the same time shares that identity and must
use its own `--namespace` to avoid sharing retention.
### Offsite retention
Retention runs after a successful upload and applies only within the selected
namespace to keys matching `<yyyymmddThhmmssZ>-<8 lowercase hex>.tar.gz` with a
valid UTC timestamp. Other objects and namespaces, including the `owner.json`
claim, are never deleted. Retention checks ownership again before pruning.
The policies form a union: a backup retained by any policy stays. Each policy
selects the newest backup in its most recent nonempty calendar buckets; days
start at midnight UTC, weeks start Monday UTC, and months follow the UTC
calendar. Missing periods do not consume a bucket. The newest backup is always
kept, including when every supplied count is zero. Counts must be nonnegative
integers. Without any `--keep-*` flags, retention deletes nothing.
```bash
openclaw backup create --to offsite --namespace gateway --keep-daily 7 --keep-weekly 4 --keep-monthly 12
```
### List and verify remote archives
```bash
openclaw backup list --from offsite
openclaw backup list --from offsite --namespace gateway
openclaw backup list --from offsite --namespace gateway --json
openclaw backup verify --from offsite --namespace gateway latest
openclaw backup verify --from offsite --namespace gateway 20260930T120000Z-a1b2c3d4.tar.gz
```
`list` requires `--from <location>` and accepts `--namespace <name>`. It lists
matching backup keys newest first, with plaintext and stored sizes. Without
`--namespace`, it also lists available namespaces under `backups/` with their
claim hostnames, helping you locate backups from another machine. `verify`
accepts those same options and either a listed key or `latest`, which selects
the newest timestamp in the key.
Use the key relative to the namespace, without the `backups/<namespace>/` prefix.
Remote verification downloads and decrypts into managed scratch, applies the
same archive verification as a local file, and removes the scratch copy.
Listing, verifying, and restoring remote archives are read-only at the location;
they neither require nor replace the namespace claim, including on a new machine.
Without `--from`, `verify` and `restore` continue to accept local archive paths.
## Restore a full archive
Restore a complete archive into a fresh staging directory without touching the
@ -66,8 +196,14 @@ live state directory:
```bash
openclaw backup restore <archive.tar.gz> --target <fresh-directory>
openclaw backup restore --from offsite --namespace gateway latest --target <fresh-directory>
```
With `--from <location>`, the archive argument is a listed key or `latest`.
`--namespace <name>` defaults to the sanitized hostname. The command downloads
and decrypts the selected archive into managed scratch before the same local
verification and restore flow.
The target must not exist or must be an empty directory, and it cannot be inside
the live state directory or any configured live agent directory. Restore
verifies the archive and its SQLite databases before creating or writing the
@ -295,23 +431,111 @@ not write a second set of table dumps.
## Schedule backups
Provision one Gateway-owned automation with a fixed name:
Provision one Gateway-owned automation per mode. Choose `--to <location>` for
offsite archives or `--repository <path>` for Git database backups:
```bash
openclaw backup enable --to offsite --every 24h --keep-daily 7 --keep-weekly 4 --keep-monthly 12
openclaw backup enable --repository ~/Backups/openclaw-git --every 24h --push
```
The interval defaults to `24h` when `--every` is omitted. An explicitly empty or whitespace-only interval is rejected before a schedule is created or updated.
The default scope is every database. Use `--global-only` or `--agent <id>` to narrow it, and add `--exclude-secrets` for a redacted history. Pushed schedules (`--push`) redact credential-bearing tables and secret-prefixed machine-state rows by default because an unattended recurring push retains them durably in remote history; pass `--include-secrets` for explicit full-fidelity remote backups (restores from redacted history need device re-pairing and provider re-authentication). `--push` also requires the repository to already have an `origin` remote. Re-running `backup enable` updates the existing automation instead of creating a duplicate. `openclaw backup disable` removes it; disabling an already-missing job is a successful no-op. Backup scheduling currently requires a local Gateway because the command job runs on the Gateway host; for a remote Gateway, create the cron job manually with `openclaw cron add`.
Offsite schedules accept `--namespace <name>`, `--claim-namespace`, `--no-include-workspace`, and
`--keep-daily`, `--keep-weekly`, and `--keep-monthly`. They use the same archive,
encryption, and [retention rules](/cli/backup#offsite-retention) as `backup create --to`.
The location and its secrets must be accessible to the Gateway process. There
is no retained local archive from scheduled offsite runs.
`--claim-namespace` is stored in the schedule's command only when explicitly
passed to `backup enable --to`; each scheduled run can then take over the
namespace. Omit it for normal ownership checks. Re-enable the schedule without
the flag when continuing takeover authority is no longer needed.
For Git schedules, the default scope is every database. Use `--global-only` or
`--agent <id>` to narrow it, and add `--exclude-secrets` for a redacted history.
Pushed schedules (`--push`) redact credential-bearing tables and secret-prefixed
machine-state rows by default because an unattended recurring push retains them
durably in remote history; pass `--include-secrets` for explicit full-fidelity
remote backups. Restores from redacted history need device re-pairing and
provider re-authentication. `--push` also requires the repository to already
have an `origin` remote. Git-only flags cannot be combined with `--to`.
Re-running `backup enable` updates the job for the selected mode instead of
creating a duplicate. Offsite and Git jobs can coexist. Existing Git jobs retain
their declaration key `openclaw-backup-scheduled`; offsite jobs use
`openclaw-backup-offsite-scheduled`.
```bash
openclaw backup disable --offsite
openclaw backup disable --git
openclaw backup disable
```
`--offsite` removes only the offsite job; `--git` removes only the Git job.
Omitting the selector removes both. Disabling an already-missing job is a
successful no-op. Enabling and disabling require a local Gateway because the
command job runs on the Gateway host; for a remote Gateway, create the cron
job manually with `openclaw cron add`.
Disabling a schedule finds the managed automation across all list pages, even after renaming it. Unrelated automations with the same name are left in place.
## Recorded runs and freshness
Every real archive, SQLite snapshot, and Git create attempt records a compact outcome in the existing shared state database. Dry runs are not recorded. The log retains the newest 200 attempts, so frequent schedules remain bounded.
Every real archive, SQLite snapshot, and Git create attempt records a compact
outcome in the existing shared state database. External jobs can also report
their outcomes. Dry runs are not recorded. The log retains the newest 200
attempts plus the newest attempt and newest successful result for every backup
kind and target, including the namespace for offsite backups. Frequent schedules cannot evict an infrequent destination's
last attempt or last success; history stays bounded by the recent window and
the number of distinct targets.
Git history is grouped by repository. Local archives and SQLite snapshots
without a named target share a bounded history group for their backup kind.
`openclaw status` shows one `Backups` overview row, and `openclaw status --json` includes the latest attempt and latest successful run. `openclaw doctor` prints an informational hint when no successful backup is recorded or the newest successful backup is more than 14 days old. Recording is best-effort: a record-write failure prints a warning but never changes a successful backup into a failed command.
Successful offsite outcomes include the location name, provider, location identity,
key, namespace, plaintext archive bytes, and stored bytes. Runs with retention
also record kept/deleted counts. Storage encryption can make stored bytes larger
than plaintext bytes. Failed offsite attempts also record the namespace they tried.
`openclaw status` shows the newest offsite result alongside the backup overview;
`openclaw status --json` includes recorded freshness. `openclaw doctor` prints an
informational hint when no successful backup is recorded or the newest success
is more than 14 days old. It also flags an enabled offsite schedule whose newest
attempt failed or whose newest success is older than three times its interval,
naming the location and `openclaw storage test <name>` as the next check. Offsite
health matches both the location and the schedule's namespace. Older records
without a namespace remain readable but cannot satisfy a namespaced schedule.
Gateway RPC `backup.status` requires operator read scope. It returns the newest
attempt and success per backup kind, target, and offsite namespace from the whole retained ledger,
configured backup schedules with their next run, and the configured storage
locations. Local archives and SQLite snapshots without a named target use one
status group per kind, displaying the newest attempt's archive path.
Listing configuration does not probe storage. The Control UI's
Backups section on the Systems landing and Gateway host views uses this status and provides a **Check** action per location
through `storage.locations.probe`.
Doctor uses the same retained history, so per-target health survives more than
200 newer outcomes from other jobs.
Recording is best-effort: a record-write failure prints a warning but never
changes a successful backup into a failed command. Recording uses an existing
shared state database; it does not create a missing database.
### Record external backup jobs
Use `backup record` after a host-level backup job, such as a restic timer, to
include its outcome in backup status and Doctor freshness:
```bash
openclaw backup record --status ok --target host-restic --bytes 1048576
openclaw backup record --status failed --target host-restic --error "Backup destination unavailable"
```
`--status ok|failed` and `--target <label>` are required. `--bytes <n>` optionally
records a nonnegative integer byte count; `--error <text>` records a diagnostic.
`--json` emits a machine-readable result. The command adds a `kind: "external"`
outcome using the same best-effort recording semantics as built-in backups. It
does not run or verify the external backup. Use a stable target label and run
it against the Gateway's state directory so successive outcomes appear together.
## What gets backed up
@ -452,7 +676,7 @@ Local edits inside a managed `dev/` checkout are developer source, not OpenClaw
Discovery reads shared state through an online SQLite snapshot so concurrent writers do not make a valid config appear invalid. If the state cannot be read, resolve the reported error and retry backup.
`--only-config` still works when the config is malformed or state discovery fails. It saves the active JSON config file alone, without parsing it or including its dependencies.
Local `--only-config` still works when the config is malformed or state discovery fails. It saves the active JSON config file alone, without parsing it or including its dependencies. Uploading it with `--to` additionally requires readable storage configuration.
## Size and performance
@ -480,7 +704,9 @@ Scratch observed by the scan that disappears before cleanup is recorded as
already reclaimed, without a warning or a claim that this pass removed it.
`openclaw doctor` reports scratch in the active temporary directory and recorded
archive destination directories. `openclaw doctor --fix` removes recognized
archive destination directories. When no backup ledger exists, recorded-location
discovery returns no directories and does not create state. Inspection stays quiet
when there is no scratch to report. `openclaw doctor --fix` removes recognized
scratch whose lifetime transaction has ended. Unknown contents, symbolic links,
and legacy directories without a lifetime token are preserved with guidance for
inspection. Older releases do not create these tokens, so stop older backup
@ -494,3 +720,5 @@ later pass can finish partial cleanup even after the lifetime token is gone.
- [CLI reference](/cli)
- [Migrating an OpenClaw install](/install/migrating)
- [Restore a full archive](/install/backups#restore-a-full-archive)
- [Storage locations](/concepts/storage-locations)
- [Storage CLI](/cli/storage)

67
docs/cli/storage.md Normal file
View file

@ -0,0 +1,67 @@
---
summary: "List, initialize, and test configured storage locations"
title: "storage"
read_when:
- You are connecting a disk or object store as a storage location
- You need to check storage access or encryption keys
---
# `openclaw storage`
Manage named destinations configured under `storage.locations`. See
[Storage locations](/concepts/storage-locations) for configuration, encryption,
and provider contracts.
```bash
openclaw storage list
openclaw storage init archive
openclaw storage test archive
```
All three commands accept `--json`, including before the subcommand:
```bash
openclaw storage list --json
openclaw storage --json init archive
openclaw storage test archive --json
```
## List
`storage list` shows each configured location, its provider, its display target
when available, and its probe state.
Probing reads the location marker and checks access and the encryption key. It
does not initialize locations or write test objects. Providers that report
capacity include free and total bytes in JSON output.
## Initialize
`storage init <name>` writes the location's identity and encryption marker with
a create-only operation. Repeating it with the same encryption configuration
verifies the existing marker; it does not replace it. A wrong passphrase fails.
For a `filesystem` location, the configured absolute path must already exist
and be a directory. OpenClaw never creates this root directory. Confirm that an
external disk is mounted before initializing its location.
Initialization is explicit so a disconnected disk or an empty mount point cannot
silently become a new destination. If a runtime operation reports a missing
marker, reconnect the disk or check the bucket and prefix. Initialize the location
only if it is new. For example:
> Storage location "r2test" (r2://bucket/prefix) has no initialization marker. If this is a new location, run `openclaw storage init r2test`; otherwise reconnect the disk or check the bucket and prefix.
## Test
`storage test <name>` writes a small, unique `.openclaw-probe-<uuid>` object at the location root, reads
it back through the configured encryption layer, verifies every byte, and deletes
the object. It requires an initialized location and never initializes one.
No probe directory is created on filesystem locations.
A successful JSON result includes `state: "ok"` and `sizeBytes`. A failed write,
read-back verification, or cleanup fails the command. If a provider is unreachable
during cleanup, reconnect it before removing any leftover probe objects.
Storage locations can contain credentials and private conversations. Use
encryption unless the destination is already protected and you deliberately
choose `encryption: "none"`.

161
docs/concepts/storage-locations.md Normal file
View file

@ -0,0 +1,161 @@
---
summary: "Named storage destinations, explicit initialization, encryption, and provider configuration"
title: "Storage locations"
read_when:
- Configuring an external disk or another storage destination
- Choosing storage encryption and initializing a location
- Diagnosing unavailable storage or a wrong encryption key
---
# Storage locations
A storage location names a destination for OpenClaw artifacts. Core storage owns
location identity, encryption, and health checks. Providers transfer objects, and
each consumer decides what to store and retain. Configuring a location does not
schedule a backup or move existing data.
The built-in `filesystem` provider supports an existing directory, including an
external disk or a mounted network filesystem. Additional providers come from
plugins. Referencing a bundled provider in config enables its owner plugin through
the normal plugin policy; an explicit disable still applies.
For Cloudflare R2 object storage, follow the
[Cloudflare plugin setup](/plugins/cloudflare) to create a bucket, configure
SecretRefs, and initialize an `r2` location.
## Configure and initialize a directory
Mount the intended disk and create the destination directory on it before
initializing storage. OpenClaw never creates the configured root directory. Use
an absolute path as seen by the process running the CLI or Gateway.
Set `OPENCLAW_STORAGE_PASSPHRASE` in that process's environment and keep a recoverable
copy of its value in your secret manager. Add a location to your config:
```json5
{
storage: {
locations: {
archive: {
provider: "filesystem",
settings: { path: "/mnt/archive/openclaw" },
encryption: {
passphrase: {
source: "env",
provider: "default",
id: "OPENCLAW_STORAGE_PASSPHRASE",
},
},
},
},
},
}
```
Initialize the destination explicitly, then verify a write/read/delete cycle:
```bash
openclaw storage init archive
openclaw storage test archive
openclaw storage list --json
```
Initialization writes `openclaw-storage.json` at the location root. Running `init`
again with the same encryption settings and passphrase is safe. Runtime operations
never create this marker: an empty mountpoint must not silently become storage on
the system disk.
## Configuration reference
Storage is optional; omitting `storage` or `storage.locations` defines no locations.
Each key under `storage.locations` is a name matching
`[a-z0-9][a-z0-9-]{0,62}`: 1–63 lowercase letters, digits, or hyphens, starting with
a letter or digit.
| Key | Required | Meaning |
| ------------------------------------------------ | -------------------------- | ------------------------------------------------------------------------------------------ |
| `storage.locations.<name>.provider` | Yes | Nonempty provider id; `filesystem` is built in. |
| `storage.locations.<name>.settings` | Yes | Provider-owned JSON object, validated before opening a backend. |
| `storage.locations.<name>.encryption` | Yes | `{ passphrase: SecretInput }` or the explicit string `"none"`. |
| `storage.locations.<name>.encryption.passphrase` | When encryption is enabled | Passphrase string or [SecretRef](/gateway/secrets/secretref-contract); prefer a reference. |
The filesystem provider accepts `settings: { path: "/absolute/existing/directory" }`.
It refuses a missing root and never overwrites an existing object key. Its probe
reports free and total filesystem space when available.
Provider settings must be finite, bounded JSON: at most 32 nesting levels, 4,096
values, 512 keys per object, string lengths of 65,536, and 256 KiB when serialized.
Secret-bearing settings, including `accessKeyId`, `secretAccessKey`, and nested
credentials, must use valid SecretRefs. Provider-specific validation can impose
additional constraints. SecretRefs remain references until the provider requests
their values through the core secret resolver.
## Choose encryption deliberately
With a passphrase, core storage encrypts object streams before the provider receives
them and decrypts them when read. The `OCSTOR1` format uses scrypt to derive a master
key and authenticated AES-256-GCM segments with a separate key for every object.
An incorrect passphrase produces `wrong-key` and refuses access. Changing the
configured passphrase does not re-encrypt existing data.
Keep both the passphrase and the location marker. Losing either can make encrypted
objects unreadable. Object names and the initialization marker remain visible to
the storage provider; encryption protects object contents.
Set `encryption: "none"` only as an explicit operator choice, such as a destination
already protected by disk encryption. **Backups can contain credentials.** Without
storage encryption, anyone who can read the destination can read the stored bytes.
## Share a location across installations
Backups use `backups/<namespace>/` within a location. The namespace defaults to
the sanitized hostname; use `--namespace <name>` to choose a stable, distinct
name for each installation sharing the destination.
The first upload creates an `owner.json` claim containing the installation's
durable Gateway device ID, hostname, and claim time. It uses the same location
encryption settings as the archives. Backup creation checks the claim before
archiving and again before retention. A different device ID refuses the backup
and records a failed attempt, preventing a hostname collision from sharing
retention. Retention never deletes the claim.
Pass `--claim-namespace` to `backup create --to <location>` to deliberately take
over a namespace, for example after moving to new hardware. `backup enable --to`
also accepts the flag and stores it in the scheduled command only when explicitly
passed; those scheduled runs can then replace another installation's claim.
Prefer a separate namespace when both installations will continue running.
Recovery remains read-only: `backup list`, `backup verify`, and `backup restore`
with `--from <location>` do not require or change the claim. Listing without
`--namespace` also shows available namespaces and their claim hostnames. A
restored installation carries its original device identity and can continue its
namespace. A cloned copy running concurrently shares that identity and must use
its own `--namespace` to avoid sharing retention.
Backup status and Doctor preserve the newest attempt and newest successful result
for every backup kind and target alongside the newest 200 global attempts. A busy
job cannot hide an infrequent destination's last result. See
[Backups](/install/backups#copy-backups-offsite) and the
[Backup CLI](/cli/backup#recorded-runs-and-freshness) for commands and status details.
## Diagnose a location
`openclaw storage list` probes configured locations. `openclaw storage test <name>`
also writes a temporary `.openclaw-probe-<uuid>` object at the location root, reads and verifies it, then
deletes it. Every storage command supports `--json`; see the
[CLI reference](/cli/storage).
| State | Next step |
| --------------- | --------------------------------------------------------------------------------------------------------------------------------------- |
| `ok` | The marker and encryption identity are valid, and the backend probe succeeded. |
| `unavailable` | Reconnect the disk or restore access to the configured destination. Check that the CLI and Gateway see the same path and credentials. |
| `uninitialized` | Confirm this is the intended new destination, then run `openclaw storage init <name>`. Never initialize an unexpected empty mountpoint. |
| `wrong-key` | Restore the original passphrase or correct the SecretRef. Do not replace the marker to hide the mismatch. |
| `error` | Read the returned message and correct provider settings, permissions, or the reported backend failure. |
Gateway clients can list configured locations with `storage.locations.list` without
storage I/O, and request a health check with `storage.locations.probe { name }`.
Both require operator read scope. Initialization remains an explicit CLI operation.
See [Configuration reference](/gateway/configuration-reference) for other config
domains and [Secrets](/gateway/secrets) for secret provider setup.

3
docs/docs.json
View file

@ -1646,6 +1646,7 @@
"pages": [
"concepts/agent-workspace",
"concepts/managed-worktrees",
"concepts/storage-locations",
"concepts/oauth",
"start/bootstrapping",
"concepts/experimental-features"
@ -1821,6 +1822,7 @@
"pages": [
"plugins/admin-http-rpc",
"plugins/beam",
"plugins/cloudflare",
"plugins/geolocation",
"plugins/logbook",
"plugins/oc-path",
@ -2757,6 +2759,7 @@
"cli/openclaw",
"cli/reset",
"cli/setup",
"cli/storage",
"cli/uninstall",
"cli/update",
{

1
docs/gateway/configuration-reference.md
View file

@ -38,6 +38,7 @@ Dedicated deep references:
- [Configuration — browser, UI, and desktop](/gateway/config-browser-ui-desktop) — browser automation, Control UI presentation, and desktop or paired-node config.
- [Configuration — gateway](/gateway/config-gateway) — gateway config: bind, auth, roles, Control UI, terminal, remote, nodes, TLS, and reload.
- [Configuration — cloud worker environments](/gateway/config-cloud-workers) — cloud worker profiles under `cloudWorkers`, including Crabbox and static SSH development.
- [Storage locations](/concepts/storage-locations) — named storage destinations under `storage.locations`, initialization, and encryption.
- [Configuration — hooks](/gateway/config-hooks) — hook config: HTTP contract, agent payload, session policy, mapping, retries, and Gmail.
- [Configuration — environment, secrets, and includes](/gateway/config-secrets-env) — environment variables, secret providers, auth storage, and `$include` config splitting.
- [Configuration — audit, logging, diagnostics, and telemetry](/gateway/config-observability) — observability config: audit, logging, diagnostics, and telemetry keys.

16
docs/help/testing-updates-plugins.md
View file

@ -249,6 +249,22 @@ Available scenarios: `base`, `acpx-openclaw-tools-bridge`, `feishu-channel`,
fixtures but excludes the expensive `sqlite-volume` scenario. Use
`OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS=far-reaching` to include it.
The opt-in `backup-schedule` scenario uses the published `openclaw@2026.9.7`
CLI to initialize a Git backup repository, enable its Gateway-owned 24-hour
schedule, and record one Git backup and one archive backup. The published updater
installs the source-pinned candidate tarball. After non-interactive Doctor and
Gateway startup, the scenario checks the original schedule declaration and argv,
both old ledger rows through `backup.status`, the status backup line, Doctor
errors, and HTTP readiness. It also requires that `storage.locations` stays
absent and the Cloudflare plugin stays inactive. This manual/release scenario
is excluded from aggregate aliases and per-PR CI.
```bash
OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS=openclaw@2026.9.7 \
OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS=backup-schedule \
pnpm test:docker:published-upgrade-survivor
```
The `custom-plugin-siblings` scenario starts from published 2026.9.4 or later
with an enabled custom memory plugin importing `../shared/value.mjs` from both
its runtime entry and Doctor config-repair contract. It runs the published

152
docs/install/backups.md
View file

@ -1,4 +1,5 @@
---
doc-schema-version: 1
summary: "Back up OpenClaw state: archives, per-database snapshots, scheduling, offsite copies, and continuous replication"
read_when:
- You want a backup routine for an OpenClaw install instead of a one-off archive
@ -39,6 +40,7 @@ committed state safely.
## Choose a path
- One-off state and workspace archive: `openclaw backup create`.
- An archive on a disk or object store: `openclaw backup create --to <location>`.
- One database, compact and verified: `openclaw backup sqlite create`.
- Versioned and incremental by content: `openclaw backup git create`.
- Regular protection: provision the Gateway-owned backup automation.
@ -147,11 +149,29 @@ recovery artifact.
## Schedule backups
The recommended schedule is one Gateway-owned automation. This example backs
up the shared database and every configured agent database daily, including
custom agent roots, and pushes the current branch to `origin`.
Pushing requires the repository to have an `origin` remote first, so
initialize it once before enabling a pushed schedule:
After [configuring and initializing a storage location](/install/backups#copy-backups-offsite),
enable a Gateway-owned daily archive backup:
```bash
openclaw backup enable --to offsite --every 24h --keep-daily 7 --keep-weekly 4 --keep-monthly 12
```
The Gateway creates, verifies, and uploads each archive using the location's
encryption settings. Add `--no-include-workspace` to omit workspace files, or
`--namespace <name>` to use a stable namespace instead of the sanitized hostname.
Keep the location's credentials and encryption passphrase available to the
Gateway process. Retention is optional; without any `--keep-*` flags, backups
are never pruned. See [retention rules](/cli/backup#offsite-retention).
Each namespace belongs to the installation that first uploads to it. To deliberately
take over an existing namespace, pass `--claim-namespace` to `backup enable --to`.
The schedule retains this flag only when explicitly passed; its runs can then
replace an existing ownership claim. Prefer a separate `--namespace` for a
different installation that is still running.
For incremental database history in Git, initialize a private repository and
enable the Git schedule instead. This backs up the shared database and every
configured agent database, including custom agent roots:
```bash
openclaw backup git init --repository ~/Backups/openclaw-git --remote git@github.com:you/openclaw-backups.git
@ -170,16 +190,23 @@ remote is private; restores from redacted history require re-pairing devices
and re-authenticating providers afterward. Local (non-push) schedules keep full
fidelity so restores are complete.
Use `--global-only` or `--agent <id>` to narrow the scope. Add
`--exclude-secrets` for a redacted Git history. Re-running the command updates
the fixed scheduled job instead of creating another one. Disable it with:
Use `--global-only` or `--agent <id>` to narrow the Git scope. Add
`--exclude-secrets` for a redacted Git history.
There is one managed job per mode: offsite archives and Git backups can run
together. Re-running `backup enable` updates the job for the selected mode,
including its destination. Existing Git schedules continue to be recognized.
The interval defaults to `24h`. Disable one mode or both:
```bash
openclaw backup disable --offsite
openclaw backup disable --git
openclaw backup disable
```
The Gateway must be reachable while enabling or disabling the schedule. There
is no local fallback scheduler.
Enabling or disabling requires a reachable local Gateway, because jobs run on
the Gateway host. There is no local fallback scheduler. For a remote Gateway,
create the job explicitly with `openclaw cron add` on that host.
As an alternative, use your platform scheduler directly. A nightly cron
example that snapshots the control-plane database and the `main` agent
@ -196,24 +223,105 @@ emits one machine-readable result per run, so the log doubles as a backup
audit trail. Prune old snapshot directories on your own retention schedule.
Every non-dry-run archive, local SQLite snapshot, and Git backup attempt is
also recorded in the shared state database. `openclaw status` shows the newest
attempt, and `openclaw doctor` suggests a one-off or scheduled backup when no
successful run is recorded or the newest success is more than 14 days old.
also recorded in the shared state database. Host-level jobs can report their
outcome with `openclaw backup record`; see
[external backup jobs](/cli/backup#record-external-backup-jobs).
`openclaw status` shows the newest backup attempt and offsite result. The
Control UI's Backups section on the Systems landing and Gateway host views shows each target's last success, size, destination,
latest failure, and next scheduled run. Its storage location **Check** action
probes access without writing a backup. `openclaw doctor` keeps the 14-day
freshness hint and also flags an offsite schedule after a failed attempt or
when its last success is older than three schedule intervals. Diagnose that
destination with `openclaw storage test <name>`.
The recorded history keeps the newest 200 attempts plus the newest attempt and
newest successful result for every backup kind and target. Status and Doctor
retain an infrequently used destination's last outcome even when another job
produces more than 200 newer results.
## Copy backups offsite
Archives and snapshot repositories are plain files, so any sync tool works.
An `rclone` example targeting an S3-compatible bucket:
Use a named [storage location](/concepts/storage-locations) for an external disk,
mounted network directory, or plugin-provided object store. The built-in
`filesystem` provider uses an existing directory; the
[Cloudflare plugin](/plugins/cloudflare) provides R2 storage. Configure the
location and its encryption first, then explicitly initialize the intended
destination:
```bash
rclone sync ~/Backups/openclaw-sqlite remote:openclaw-backups/sqlite
openclaw storage init offsite
openclaw storage test offsite
openclaw backup create --to offsite
```
Because every archive and local snapshot is a full copy, offsite syncs re-upload
each new backup in full. Deduplicating backup tools such as `restic` reduce
storage at the destination but still read full snapshots as input. When
upload size per backup matters, use Git-backed snapshots or continuous
replication.
The backup command checks the location before creating the archive, verifies
the archive locally, uploads it, and confirms its stored size. A missing
initialization marker refuses the backup: reconnect the disk or check the bucket
and prefix, or initialize
the location only if it is new. Runtime backups never initialize a location
or create a missing filesystem root. See [Storage CLI](/cli/storage).
By default, the local archive lives only in managed scratch space and is
removed after the upload. Add `--output ~/Backups/openclaw` to retain a local
copy. Local copies are plaintext `.tar.gz` archives even when the storage
location encrypts uploaded bytes; protect both destinations accordingly.
Backups are stored under `backups/<namespace>/`, where the namespace defaults
to the sanitized hostname. Use a distinct explicit namespace for each installation
sharing a destination:
```bash
openclaw backup create --to offsite --namespace gateway --keep-daily 7 --keep-weekly 4 --keep-monthly 12
openclaw backup list --from offsite --namespace gateway
openclaw backup verify --from offsite --namespace gateway latest
openclaw backup restore --from offsite --namespace gateway latest --target ./restored-openclaw
```
The first upload claims the namespace for the installation's durable Gateway
device identity. Its `owner.json` contains the device ID, hostname, and claim
time, and uses the location's encryption settings. OpenClaw checks ownership
before archiving, at archive publication, and before each retention deletion.
A different device identity causes a failed attempt with the owner's hostname and abbreviated device ID, even if
both machines use the same hostname.
Choose another `--namespace` for a separate installation. To deliberately take
over a stopped or retired installation's namespace, for example after moving to
new hardware, run:
```bash
openclaw backup create --to offsite --namespace gateway --claim-namespace
```
The displaced installation is rejected at its next publication or deletion.
Object stores cannot make an object's write conditional on a separate ownership
claim, so a residual provider round-trip window remains between the final check
and the effect. Stop the old installation before taking over; use a separate
namespace for installations that run concurrently.
Retention runs only after a successful upload. It keeps the newest backup in
each selected UTC day, week, or month, combining the policies and always
preserving the newest backup. It never deletes other namespaces or objects
whose keys do not match the backup filename pattern, including `owner.json`.
No retention flags means
no deletion; see [Backup CLI](/cli/backup#offsite-retention) for exact rules.
Remote verification and restore download and decrypt into managed scratch,
then use the same archive checks as local files. Restore still stages into a
fresh target; follow [Restore a full archive](/install/backups#restore-a-full-archive) before
activating the result. When recovering on another host, specify the original
namespace and retain the original encryption passphrase and location marker.
`backup list --from offsite` without `--namespace` also lists available
namespaces and their claim hostnames so you can find the original data.
Listing, verifying, and restoring never require or change namespace ownership.
A restored installation carries the original device identity and can continue
using its namespace. A cloned copy running concurrently shares that identity;
give it its own `--namespace` so the two copies do not share retention.
Each archive is a full copy. For large installs where upload size matters,
use Git-backed snapshots or continuous replication. Plain local archives and
snapshot repositories remain ordinary files that external backup tools can copy.
## Versioned backups to a Git repository
@ -432,6 +540,8 @@ first with `openclaw database preflight`; see
- [Agent workspace](/concepts/agent-workspace#git-backup-recommended-private) for keeping workspace files in a private git repository
- [Backup CLI reference](/cli/backup)
- [Cloudflare Containers](/install/cloudflare) — continuous Litestream replication to R2 for an ephemeral container deployment
- [Cloudflare plugin](/plugins/cloudflare) — R2 storage locations for archive backups
- [Database schemas](/reference/database-schemas)
- [Migrating between machines](/install/migrating)
- [Storage locations](/concepts/storage-locations)
- [Updating](/install/updating)

140
docs/plugins/cloudflare.md Normal file
View file

@ -0,0 +1,140 @@
---
summary: "Configure Cloudflare R2 as an encrypted OpenClaw storage location"
title: "Cloudflare"
read_when:
- You want to store OpenClaw artifacts in Cloudflare R2
- You need to configure an R2 bucket, credentials, or jurisdiction
- You are diagnosing R2 storage access
---
# Cloudflare
The bundled Cloudflare plugin provides `r2` storage locations through Cloudflare's
S3-compatible API. OpenClaw handles location identity and encryption; the plugin
transfers objects to your bucket. Configuring a location does not schedule backups
or move existing data.
## Create a bucket and credentials
1. In the Cloudflare dashboard, open **R2 object storage** and create a private
bucket, such as `openclaw-artifacts`. Bucket names must contain 3–63 lowercase
letters, digits, or hyphens, and cannot begin or end with a hyphen.
2. Record your Cloudflare account ID and the bucket's jurisdiction, if any.
3. From R2's **Account Details**, select **Manage** next to **API Tokens**, then
**Create Account API token**.
4. Select **Object Read & Write** and limit access to the bucket you created.
5. Save the **Access Key ID** and **Secret Access Key** in your secret manager.
Cloudflare shows the secret access key only once.
Use the R2 S3 credentials from this flow. See Cloudflare's
[bucket creation guide](https://developers.cloudflare.com/r2/buckets/create-buckets/)
and [R2 token guide](https://developers.cloudflare.com/r2/api/tokens/).
## Configure a location
Make the saved credentials available as `R2_ACCESS_KEY_ID` and
`R2_SECRET_ACCESS_KEY` in the environment that runs the CLI and Gateway. Create a
separate encryption passphrase, keep a recoverable copy in your secret manager,
and provide it as `OPENCLAW_STORAGE_PASSPHRASE` in that environment.
Add this to your OpenClaw config, replacing the example account ID and bucket:
```json5
{
storage: {
locations: {
offsite: {
provider: "r2",
settings: {
accountId: "00000000000000000000000000000000",
bucket: "openclaw-artifacts",
prefix: "openclaw",
accessKeyId: {
source: "env",
provider: "default",
id: "R2_ACCESS_KEY_ID",
},
secretAccessKey: {
source: "env",
provider: "default",
id: "R2_SECRET_ACCESS_KEY",
},
},
encryption: {
passphrase: {
source: "env",
provider: "default",
id: "OPENCLAW_STORAGE_PASSPHRASE",
},
},
},
},
},
}
```
R2 credential settings require [SecretRefs](/gateway/secrets/secretref-contract);
plaintext credential strings are rejected. You can use another configured secret
provider instead of environment variables. A Gateway running as a service needs
the values in its own environment, not just in your interactive shell.
Referencing `provider: "r2"` automatically enables the bundled `cloudflare` plugin
under the normal plugin policy. Explicit disablement and deny rules still apply.
## Initialize and test
Confirm the bucket and prefix, then initialize the location and verify a complete
write/read/delete cycle:
```bash
openclaw storage init offsite
openclaw storage test offsite
openclaw storage list --json
```
Initialization writes the location marker at
`openclaw/openclaw-storage.json` for the example above. The displayed target is
`r2://openclaw-artifacts/openclaw`. A successful test confirms that it wrote, read,
verified, and deleted its probe object; add `--json` for `state: "ok"`.
Keep the marker and encryption passphrase: losing either can make encrypted
objects unreadable. R2 health checks verify bucket access without reporting free
or total space.
## Settings
All fields below belong to `storage.locations.<name>.settings`.
| Field | Required | Meaning |
| ----------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `accountId` | Yes | Cloudflare account ID: exactly 32 lowercase hexadecimal characters. |
| `bucket` | Yes | Existing R2 bucket name, following the naming rules above. |
| `prefix` | No | Object-key prefix. Use slash-separated segments containing only letters, digits, `.`, `_`, and `-`; no empty, `.` or `..` segments, or leading/trailing slash. Omit it to use the bucket root. |
| `jurisdiction` | No | `"eu"` or `"fedramp"`, matching the bucket's jurisdiction. Omit for the default endpoint. |
| `accessKeyId` | Yes | SecretRef for the R2 access key ID. |
| `secretAccessKey` | Yes | SecretRef for the R2 secret access key. |
| `sessionToken` | No | SecretRef for the session token when using R2 temporary credentials. |
The plugin uses region `auto`. It chooses
`https://<accountId>.r2.cloudflarestorage.com` by default,
`https://<accountId>.eu.r2.cloudflarestorage.com` for `"eu"`, or
`https://<accountId>.fedramp.r2.cloudflarestorage.com` for `"fedramp"`.
No custom endpoint is required. A prefix is a namespace within the bucket, not a
separate permission boundary; the token remains scoped to the bucket.
## Troubleshooting
For a 401 or 403 error, check that the R2 token has **Object Read & Write** on the
configured bucket and that both SecretRefs resolve in the process running the
command. For temporary credentials, also verify that the session token is present
and has not expired.
If the bucket is missing, create it in the configured account or correct
`accountId`, `bucket`, and `jurisdiction`. OpenClaw does not create buckets.
If the location has no initialization marker, confirm that the prefix is correct before
running `openclaw storage init <name>`. Changing the prefix selects a different
location root. For `wrong-key`, restore the original encryption passphrase;
replacing the marker does not recover encrypted data.
See [Storage locations](/concepts/storage-locations) for encryption and location
lifecycle, and the [storage CLI reference](/cli/storage) for command output.

8
docs/plugins/manifest/capabilities.md
View file

@ -46,6 +46,7 @@ model tools or replace host tool authorization. See [Code Mode executors](/tools
"webFetchProviders": ["firecrawl"],
"webSearchProviders": ["gemini"],
"workerProviders": ["example-worker"],
"storageProviders": ["example-storage"],
"usageProviders": ["acme-ai"],
"migrationProviders": ["hermes"],
"gatewayMethodDispatch": ["authenticated-request"],
@ -77,6 +78,7 @@ Each list is optional. For `speechProviders` and `realtimeVoiceProviders`, list
| `webFetchProviders` | `string[]` | Web-fetch provider ids this plugin owns. |
| `webSearchProviders` | `string[]` | Web-search provider ids this plugin owns. |
| `workerProviders` | `string[]` | Cloud-worker provider ids this plugin owns for provisioning and profile-backed lease lifecycle. |
| `storageProviders` | `string[]` | Storage transport ids registered with `api.registerStorageProvider`; selected by named storage locations. |
| `usageProviders` | `string[]` | Provider ids whose usage-auth and usage-snapshot hooks this plugin owns. |
| `migrationProviders` | `string[]` | Import provider ids this plugin owns for [`openclaw migrate`](/cli/migrate). |
| `gatewayMethodDispatch` | `string[]` | Reserved entitlement for authenticated plugin HTTP routes that dispatch Gateway methods in-process. |
@ -96,6 +98,12 @@ Provider plugins that implement both `resolveUsageAuth` and `fetchUsageSnapshot`
Embedding providers must declare `contracts.embeddingProviders` for each adapter registered with `api.registerEmbeddingProvider(...)`. The same generic contract serves reusable vector generation and memory search. The retired `contracts.memoryEmbeddingProviders` key is no longer accepted.
Storage providers must declare each `api.registerStorageProvider(...)` id in
`contracts.storageProviders`. Core reserves `filesystem`. Config references in
`storage.locations` auto-enable bundled owners through this declaration; explicit
disablement and deny rules still apply. External providers require explicit plugin
enablement. See the [storage provider contract](/plugins/sdk-overview/capabilities#storage-providers).
Worker providers must declare each `api.registerWorkerProvider(...)` id in `contracts.workerProviders`. Registration requires `resolveAllocation`, `provision`, `inspect`, and `destroy`. The allocation resolver returns the exact operation cleanup handle and an explicit shared-host fact without creating or preparing a machine; see the [worker provider contract](/plugins/sdk-overview/capabilities#worker-providers). Core persists durable intent before calling `provision`; providers validate their settings and optional per-dispatch `machineClass` and `executionMode` before external allocation, and repeated calls with the same operation id must adopt the same lease without changing the selected mode. Providers may implement asynchronous `listMachineOptions(profile)` to expose process-stable picker metadata; omit it when machine selection is not meaningful. Machine options contain only `id`, `label`, optional positive-integer `cpu` and `memoryGb`, and optional `default`. Session-placement providers declare a closed, unique, canonically ordered `supportedExecutionModes` tuple: `["worker-turn"]`, `["remote-exec"]`, or `["worker-turn", "remote-exec"]`. Empty lists, duplicates, unknown values, and noncanonical ordering are rejected. `worker-turn` requires a node lease; `remote-exec` accepts a node lease or an SSH lease. Omission advertises no session-placement modes while leaving direct lifecycle operations available. A direct environment create supplies no session execution mode; providers use their documented default, which is `worker-turn` for Crabbox. Providers whose bounded provisioning exceeds core's five-minute default may implement `resolveProvisionTimeoutMs(profile)` and include acquisition, provider-owned setup, and cleanup in the returned positive millisecond budget. The optional `resolveDestroyTimeoutMs(profile)` supplies the equivalent budget for requested teardown and bootstrap-failure cleanup, including snapshot capture before confirmed release. Both hooks must return positive safe integers within the platform timer limit; an explicit service timeout override takes precedence. Core also persists that validated settings snapshot and passes it with `leaseId` to `inspect({ leaseId, profile })` and `destroy({ leaseId, profile })`, including after the named profile is changed or removed. Destruction is idempotent, inspection returns the closed `active` / `dormant` / `destroyed` / `unknown` status union, and SSH private-key material is referenced only through `SecretRef`. Provisioned SSH endpoints must also include a public `hostKey` from trusted provisioning output as exactly `algorithm base64`, without a hostname or comment, so core can pin the host before connecting. They may include up to 10 ordered, unique `fallbackPorts`, excluding the primary `port`; core persists those candidates and rotates among them only for idempotent probes, content-addressed transfers, receipt/lock-guarded artifact installation, convergent managed-worktree mirroring, and tunnel reconnects. Ambiguous unguarded stateful commands fail closed and are not replayed across candidates. A lease may set `sharedHost: true` when the SSH account also owns unrelated processes; core then avoids host-wide process freezing during workspace reconciliation. Omitted or `false` means a dedicated worker host. Active inspection repeats this fact so core can reconcile provider-owned isolation for leases persisted before the field existed; tunnel startup waits for that first authoritative inspection. Optional desktop metadata may advertise up to eight unique closed apps: `browser` with an absolute `executablePath` and a CDP port from 1 through 65535, or `terminal` with an absolute `executablePath`. Core rejects unknown app ids and fields and persists the validated metadata with the existing desktop record. Providers that mint dynamic identity refs may implement authoritative `resolveSshIdentity({ leaseId, profile, keyRef })`; providers without it use core's generic secret resolver. An authoritative `unknown` fences the environment and enters canonical teardown; it does not bypass the exact worker-stop acknowledgment required on shared or unknown hosts.
`contracts.gatewayMethodDispatch` accepts a single value, `"authenticated-request"`. It is an API hygiene gate for authenticated native plugin HTTP routes and registered RPC handlers that intentionally dispatch Gateway methods in-process, not a sandbox against malicious native plugins. Dispatch retains the original authenticated client and profile, applies the target method’s scopes, and never creates a synthetic caller. Use it only for tightly reviewed surfaces. RPC handlers still pass ordinary Gateway admission; this contract adds no suspension bypass. An entitled route remains reachable while Gateway root-work admission is closed only when it also declares `auth: "gateway"` and the route-specific `gatewayRuntimeScopeSurface: "trusted-operator"`; ordinary sibling routes from the same plugin remain behind the admission boundary. This keeps suspension status and resume reachable without granting the whole plugin an admission bypass. Keep parsing and response shaping bounded outside dispatch; substantive or mutating work must go through Gateway method dispatch, which owns admission and scope enforcement.

4
docs/plugins/plugin-inventory.md
View file

@ -50,7 +50,7 @@ Each entry lists the package, distribution route, and description.
## Core npm package
64 plugins
65 plugins
- **[a2a](/plugins/reference/a2a)** (`@openclaw/a2a`) - included in OpenClaw. A2A v1.0 Agent-to-Agent protocol channel plugin.
@ -78,6 +78,8 @@ Each entry lists the package, distribution route, and description.
- **[clawrouter](/plugins/reference/clawrouter)** (`@openclaw/clawrouter`) - included in OpenClaw. Adds ClawRouter model provider support to OpenClaw.
- **[cloudflare](/plugins/reference/cloudflare)** (`@openclaw/cloudflare`) - included in OpenClaw, and also from npm or ClawHub: `clawhub:@openclaw/cloudflare`. Cloudflare R2 storage for named OpenClaw storage locations.
- **[code-mode-quickjs](/plugins/reference/code-mode-quickjs)** (`@openclaw/code-mode-quickjs`) - included in OpenClaw. Hardened JavaScript execution for Code Mode using QuickJS in WebAssembly.
- **[copilot-proxy](/plugins/reference/copilot-proxy)** (`@openclaw/copilot-proxy`) - included in OpenClaw. Adds Copilot Proxy model provider support to OpenClaw.

2
docs/plugins/reference.md
View file

@ -13,7 +13,7 @@ This section holds one reference page for each OpenClaw plugin. Each page states
the package, the install route, and the surface the plugin adds.
This page is a pointer, not the index. The browsable list of all
162 generated plugin reference pages lives in
163 generated plugin reference pages lives in
[Plugin inventory](/plugins/plugin-inventory), sorted by distribution, package,
and description.

26
docs/plugins/reference/cloudflare.md Normal file
View file

@ -0,0 +1,26 @@
---
summary: "Cloudflare R2 storage for named OpenClaw storage locations."
read_when:
- You are installing, configuring, or auditing the cloudflare plugin
title: "Cloudflare plugin reference"
---
<!-- Generated file. Do not edit by hand.
Run `pnpm plugins:inventory:gen` to rebuild it. Hand-written text survives only
between the openclaw-plugin-reference:manual-start and
openclaw-plugin-reference:manual-end comment markers. -->
Cloudflare R2 storage for named OpenClaw storage locations.
## Distribution
- Package: `@openclaw/cloudflare`
- Install route: included in OpenClaw, and also from npm or ClawHub: `clawhub:@openclaw/cloudflare`
## Surface
- Contracts: `storageProviders`
## Related docs
- [cloudflare](/plugins/cloudflare)

2
docs/plugins/sdk-overview.md
View file

@ -42,7 +42,7 @@ Backend plugin APIs and ordinary plugin loading do not require that setting.
## What each page covers
- [Imports and module layout](/plugins/sdk-overview/imports) — which subpath to import from, the subpath catalog, and the internal barrel convention.
- [Capability registration](/plugins/sdk-overview/capabilities) — provider registrars plus the worker-provider and embedding runtime contracts.
- [Capability registration](/plugins/sdk-overview/capabilities) — provider registrars plus storage, worker, and embedding runtime contracts.
- [Tools and commands](/plugins/sdk-overview/tools-and-commands) — agent tools, custom commands, node-host commands, and widget presenters.
- [Infrastructure registration](/plugins/sdk-overview/infrastructure) — hooks, HTTP routes, Gateway methods, services, and the webhook and SQLite helpers.
- [Host hooks](/plugins/sdk-overview/host-hooks) — session extensions, trusted tool policies, Control UI descriptors, and runtime lifecycle.

51
docs/plugins/sdk-overview/capabilities.md
View file

@ -1,15 +1,16 @@
---
summary: "Provider, worker-provider, and embedding registration on OpenClawPluginApi"
summary: "Provider, storage, worker, and embedding registration on OpenClawPluginApi"
title: "Plugin SDK capability registration"
sidebarTitle: "Capability registration"
read_when:
- You are registering an inference, media, search, or transcript provider
- You are implementing the cloud-worker provider lifecycle
- You are registering an embedding provider
- You are implementing a storage location transport
---
The capability registrars on `OpenClawPluginApi`, and the runtime contracts a
worker or embedding provider must satisfy. Part of the
worker, storage, or embedding provider must satisfy. Part of the
[Plugin SDK overview](/plugins/sdk-overview).
## Capability registration
@ -18,6 +19,7 @@ worker or embedding provider must satisfy. Part of the
| ------------------------------------------------ | --------------------------------------------------------------------------------- |
| `api.registerProvider(...)` | Text inference (LLM) |
| `api.registerWorkerProvider(...)` | Cloud-worker lifecycle leases |
| `api.registerStorageProvider(...)` | Opaque object storage for named locations |
| `api.registerModelCatalogProvider(...)` | Model catalog rows for text and media generation |
| `api.registerAgentHarness(...)` | [Experimental](/plugins/sdk-agent-harness) native agent executor (Codex, Copilot) |
| `api.registerCliBackend(...)` | Local CLI inference backend |
@ -48,6 +50,51 @@ resolve one with this descriptor. OpenClaw rejects ambiguous or unresolved owner
persists the start or invokes the provider. Provider aliases are lookup names
only and must not be used for this declaration.
### Storage providers
Import `StorageProvider`, `StorageProviderOpenParams`, `StorageBackend`, and
`StorageObjectInfo` from `openclaw/plugin-sdk/plugin-entry`. Register a transport
with `api.registerStorageProvider(provider)` and declare its `id` in
`contracts.storageProviders`. Undeclared IDs, duplicate IDs, and the core-owned
`filesystem` ID are rejected. A configured `storage.locations.<name>.provider`
automatically enables its bundled owner, subject to explicit plugin disablement
and deny rules. External plugins still require explicit enablement.
A provider has an `id`, a `label`, optional synchronous `validateSettings(settings)`
returning a user-facing error, optional `describeTarget(settings)`, and asynchronous
`open(params)`. `describeTarget` returns a non-secret display target or `undefined`.
It must be pure and synchronous: derive the target from settings without I/O or
secret resolution. Configuration listings call it only for the built-in provider
or a provider already present in the supplied registry; they never activate a
plugin or open a backend to describe a target. Otherwise, `displayTarget` is omitted.
Core passes `open` the
location name, read-only settings, optional abort signal, and `resolveSecret(ref)`.
Resolve credentials through that callback; secret-bearing settings must contain
SecretRefs. Return a backend with a non-secret `displayTarget` and these methods:
| Method | Contract |
| --------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
| `probe({ signal }?)` | Return optional `freeBytes` and `totalBytes`. |
| `putObject(key, body, { sizeBytes, signal })` | Consume an `AsyncIterable<Uint8Array>` and return the stored byte count. Never overwrite an existing key. |
| `getObject(key, { signal }?)` | Return a byte stream, or `undefined` for an absent object. |
| `statObject(key, { signal }?)` | Return `{ key, sizeBytes, modifiedAt? }`, or `undefined`. Timestamps use milliseconds. |
| `listObjects(prefix, { signal }?)` | Stream object metadata beneath the prefix. |
| `deleteObject(key, { signal }?)` | Delete the object. |
| `close()` | Optional asynchronous resource cleanup. |
`sizeBytes`, when provided, is exact. Use atomic conditional creation when the
backend supports it; otherwise check for an existing object first and document
the race. Honor abort signals throughout streaming and keep memory bounded.
Core validates keys before calling the transport: slash-separated segments of
ASCII letters, digits, `.`, `_`, and `-`, excluding `.` and `..`, with no empty
segments, leading slash, or backslash, and a maximum of 512 bytes.
Providers store opaque bytes. Core owns location initialization, marker identity,
namespacing, encryption, and health classification; consumers own retention.
Providers must not create or interpret location markers or expose credentials in
`displayTarget` or errors. See [Storage locations](/concepts/storage-locations).
### Worker providers
Worker providers must also declare their id in `contracts.workerProviders`.

13
extensions/cloudflare/README.md Normal file
View file

@ -0,0 +1,13 @@
# Cloudflare OpenClaw plugin
Official OpenClaw plugin for Cloudflare R2 storage locations.
## Install
```sh
openclaw plugins install @openclaw/cloudflare
```
## Docs
See `docs/plugins/cloudflare.md` in the OpenClaw repository, or the published docs at <https://docs.openclaw.ai/plugins/cloudflare>, for bucket setup, credentials, and storage configuration.

13
extensions/cloudflare/api.ts Normal file
View file

@ -0,0 +1,13 @@
import type { StorageProvider } from "openclaw/plugin-sdk/plugin-entry";
import { describeR2Target, validateR2Settings } from "./settings.js";
export const r2StorageProvider: StorageProvider = {
id: "r2",
label: "Cloudflare R2",
validateSettings: validateR2Settings,
describeTarget: describeR2Target,
async open(params) {
const { openR2Backend } = await import("./runtime-api.js");
return openR2Backend(params);
},
};

141
extensions/cloudflare/cloudflare.live.test.ts Normal file
View file

@ -0,0 +1,141 @@
import { randomUUID } from "node:crypto";
import type { StorageObjectInfo } from "openclaw/plugin-sdk/plugin-entry";
import { isSecretRef } from "openclaw/plugin-sdk/secret-input";
import { isLiveTestEnabled } from "openclaw/plugin-sdk/test-live";
import { describe, expect, it } from "vitest";
import { r2StorageProvider } from "./api.js";
const accountId = process.env.OPENCLAW_LIVE_R2_ACCOUNT_ID?.trim() ?? "";
const bucket = process.env.OPENCLAW_LIVE_R2_BUCKET?.trim() ?? "";
const accessKeyId = process.env.OPENCLAW_LIVE_R2_ACCESS_KEY_ID?.trim() ?? "";
const secretAccessKey = process.env.OPENCLAW_LIVE_R2_SECRET_ACCESS_KEY?.trim() ?? "";
const sessionToken = process.env.OPENCLAW_LIVE_R2_SESSION_TOKEN?.trim() ?? "";
const describeLive =
isLiveTestEnabled() && accountId && bucket && accessKeyId && secretAccessKey
? describe
: describe.skip;
async function* bytes(value: Uint8Array): AsyncIterable<Uint8Array> {
yield value.subarray(0, 5);
yield value.subarray(5);
}
describeLive("Cloudflare R2 storage live", () => {
it("round-trips single and multipart objects without overwriting existing keys", async () => {
const prefix = `openclaw-live/${randomUUID()}`;
const credentials = new Map([
["OPENCLAW_LIVE_R2_ACCESS_KEY_ID", accessKeyId],
["OPENCLAW_LIVE_R2_SECRET_ACCESS_KEY", secretAccessKey],
["OPENCLAW_LIVE_R2_SESSION_TOKEN", sessionToken],
]);
const backend = await r2StorageProvider.open({
locationName: "r2-live",
settings: {
accountId,
bucket,
prefix,
accessKeyId: { source: "env", provider: "default", id: "OPENCLAW_LIVE_R2_ACCESS_KEY_ID" },
secretAccessKey: {
source: "env",
provider: "default",
id: "OPENCLAW_LIVE_R2_SECRET_ACCESS_KEY",
},
...(sessionToken
? {
sessionToken: {
source: "env",
provider: "default",
id: "OPENCLAW_LIVE_R2_SESSION_TOKEN",
},
}
: {}),
},
resolveSecret: async (ref) => {
const value = isSecretRef(ref) ? credentials.get(ref.id) : undefined;
if (!value) {
throw new Error("R2 live credential reference is unavailable.");
}
return value;
},
});
const objects = [
{ key: "known-size.txt", body: Buffer.from("OpenClaw R2 single upload"), knownSize: true },
{
key: "unknown-size.txt",
body: Buffer.from("OpenClaw R2 multipart upload"),
knownSize: false,
},
];
const failures: Error[] = [];
try {
expect(backend.displayTarget).toBe(`r2://${bucket}/${prefix}`);
await expect(backend.probe()).resolves.toEqual({});
for (const object of objects) {
const options = object.knownSize ? { sizeBytes: object.body.byteLength } : {};
await expect(backend.putObject(object.key, bytes(object.body), options)).resolves.toEqual({
sizeBytes: object.body.byteLength,
});
const replacement = Buffer.from("This must not replace the original");
await expect(
backend.putObject(
object.key,
bytes(replacement),
object.knownSize ? { sizeBytes: replacement.byteLength } : {},
),
).rejects.toThrow(/already exists|conflict/i);
await expect(backend.statObject(object.key)).resolves.toMatchObject({
key: object.key,
sizeBytes: object.body.byteLength,
});
const downloaded = await backend.getObject(object.key);
expect(downloaded).toBeDefined();
const chunks: Uint8Array[] = [];
for await (const chunk of downloaded!) {
chunks.push(chunk);
}
expect(Buffer.concat(chunks)).toEqual(object.body);
}
const listed: StorageObjectInfo[] = [];
for await (const object of backend.listObjects("")) {
listed.push(object);
}
expect(listed).toHaveLength(objects.length);
expect(listed).toEqual(
expect.arrayContaining(
objects.map(({ key, body }) =>
expect.objectContaining({ key, sizeBytes: body.byteLength }),
),
),
);
for (const { key } of objects) {
await backend.deleteObject(key);
await expect(backend.statObject(key)).resolves.toBeUndefined();
await expect(backend.getObject(key)).resolves.toBeUndefined();
}
} catch (error) {
failures.push(error instanceof Error ? error : new Error("R2 live operation failed."));
} finally {
try {
const cleanup = await Promise.allSettled(
objects.map(({ key }) => backend.deleteObject(key)),
);
if (cleanup.some((result) => result.status === "rejected")) {
failures.push(
new Error(
`R2 live cleanup failed; remove test objects under r2://${bucket}/${prefix}.`,
),
);
}
} finally {
try {
await backend.close?.();
} catch {
failures.push(new Error("R2 live client cleanup failed."));
}
}
}
if (failures.length > 0) {
throw new AggregateError(failures, "R2 live storage test failed.");
}
}, 120_000);
});

11
extensions/cloudflare/index.ts Normal file
View file

@ -0,0 +1,11 @@
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
import { r2StorageProvider } from "./api.js";
export default definePluginEntry({
id: "cloudflare",
name: "Cloudflare",
description: "Cloudflare R2 storage for named OpenClaw storage locations.",
register(api) {
api.registerStorageProvider(r2StorageProvider);
},
});

13
extensions/cloudflare/openclaw.plugin.json Normal file
View file

@ -0,0 +1,13 @@
{
"id": "cloudflare",
"name": "Cloudflare",
"description": "Cloudflare R2 storage for named OpenClaw storage locations.",
"categories": ["infrastructure"],
"activation": { "onStartup": false },
"contracts": { "storageProviders": ["r2"] },
"configSchema": {
"type": "object",
"additionalProperties": false,
"properties": {}
}
}

35
extensions/cloudflare/package.json Normal file
View file

@ -0,0 +1,35 @@
{
"name": "@openclaw/cloudflare",
"version": "2026.9.7",
"description": "Cloudflare R2 storage provider for OpenClaw.",
"repository": {
"type": "git",
"url": "https://github.com/openclaw/openclaw"
},
"type": "module",
"dependencies": {
"@aws-sdk/client-s3": "3.1136.0"
},
"devDependencies": {
"@openclaw/plugin-sdk": "workspace:*"
},
"openclaw": {
"extensions": ["./index.ts"],
"install": {
"clawhubSpec": "clawhub:@openclaw/cloudflare",
"npmSpec": "@openclaw/cloudflare",
"defaultChoice": "npm",
"minHostVersion": ">=2026.9.7"
},
"compat": {
"pluginApi": ">=2026.9.7"
},
"build": {
"openclawVersion": "2026.9.7"
},
"release": {
"publishToClawHub": true,
"publishToNpm": true
}
}
}

37
extensions/cloudflare/runtime-api.ts Normal file
View file

@ -0,0 +1,37 @@
import { S3Client } from "@aws-sdk/client-s3";
import type { StorageBackend, StorageProviderOpenParams } from "openclaw/plugin-sdk/plugin-entry";
import { createR2Backend } from "./s3-backend.js";
import { parseR2Settings, r2Endpoint } from "./settings.js";
export async function openR2Backend(params: StorageProviderOpenParams): Promise<StorageBackend> {
const settings = parseR2Settings(params.settings);
params.signal?.throwIfAborted();
let credentials;
try {
const [accessKeyId, secretAccessKey, sessionToken] = await Promise.all([
params.resolveSecret(settings.accessKeyId),
params.resolveSecret(settings.secretAccessKey),
settings.sessionToken ? params.resolveSecret(settings.sessionToken) : undefined,
]);
if (!accessKeyId || !secretAccessKey || (settings.sessionToken && !sessionToken)) {
throw new Error("Empty credential");
}
credentials = { accessKeyId, secretAccessKey, sessionToken };
} catch {
throw new Error(
"R2 credentials could not be resolved; check the configured SecretRefs and their secret providers.",
);
}
params.signal?.throwIfAborted();
return createR2Backend(
settings,
new S3Client({
endpoint: r2Endpoint(settings),
region: "auto",
credentials,
forcePathStyle: true,
requestChecksumCalculation: "WHEN_REQUIRED",
responseChecksumValidation: "WHEN_REQUIRED",
}),
);
}

377
extensions/cloudflare/s3-backend.test.ts Normal file
View file

@ -0,0 +1,377 @@
import {
AbortMultipartUploadCommand,
CompleteMultipartUploadCommand,
CreateMultipartUploadCommand,
DeleteObjectCommand,
GetObjectCommand,
HeadBucketCommand,
HeadObjectCommand,
ListObjectsV2Command,
PutObjectCommand,
S3ServiceException,
UploadPartCommand,
} from "@aws-sdk/client-s3";
import type { StorageObjectInfo } from "openclaw/plugin-sdk/plugin-entry";
import { describe, expect, it, vi } from "vitest";
import { createR2Backend } from "./s3-backend.js";
import type { R2Settings } from "./settings.js";
const PART_BYTES = 64 * 1024 * 1024;
const settings: R2Settings = {
accountId: "0".repeat(32),
bucket: "test-bucket",
prefix: "archive/team",
accessKeyId: { source: "env", provider: "default", id: "R2_KEY" },
secretAccessKey: { source: "env", provider: "default", id: "R2_SECRET" },
};
function serviceError(name: string, code: number) {
return new S3ServiceException({
name,
$fault: "client",
$metadata: { httpStatusCode: code },
message: "private-credential-value",
});
}
async function* bytes(...chunks: Uint8Array[]) {
yield* chunks;
}
function fixture() {
const send = vi.fn();
const destroy = vi.fn();
return { send, destroy, backend: createR2Backend(settings, { send, destroy }) };
}
describe("R2 object transport", () => {
it("conditionally creates small objects and preserves a conflicting object", async () => {
const { send, backend } = fixture();
send.mockResolvedValueOnce({}).mockRejectedValueOnce(serviceError("PreconditionFailed", 412));
expect(
await backend.putObject("backup.tar", bytes(Buffer.from("abc")), { sizeBytes: 3 }),
).toEqual({ sizeBytes: 3 });
expect(send.mock.calls[0]?.[0]).toBeInstanceOf(PutObjectCommand);
expect(send.mock.calls[0]?.[0].input).toEqual({
Bucket: "test-bucket",
Key: "archive/team/backup.tar",
Body: Buffer.from("abc"),
ContentLength: 3,
IfNoneMatch: "*",
});
await expect(
backend.putObject("backup.tar", bytes(Buffer.from("new")), { sizeBytes: 3 }),
).rejects.toThrow("use a new object key");
expect(send).toHaveBeenCalledTimes(2);
});
it.each([0, PART_BYTES])("uses conditional PutObject for known size %i", async (sizeBytes) => {
const { send, backend } = fixture();
send.mockResolvedValue({});
expect(
await backend.putObject("boundary", bytes(Buffer.alloc(sizeBytes)), { sizeBytes }),
).toEqual({ sizeBytes });
expect(send.mock.calls[0]?.[0]).toBeInstanceOf(PutObjectCommand);
expect(send.mock.calls[0]?.[0].input).toMatchObject({
ContentLength: sizeBytes,
IfNoneMatch: "*",
});
});
it.each([undefined, PART_BYTES + 3])(
"uploads ordered bounded parts with declared size %s",
async (sizeBytes) => {
const { send, backend } = fixture();
const lengths: number[] = [];
const edges: number[][] = [];
send.mockImplementation(async (command) => {
if (command instanceof CreateMultipartUploadCommand) {
return { UploadId: "upload" };
}
if (command instanceof UploadPartCommand) {
const body = command.input.Body;
if (!(body instanceof Uint8Array)) {
throw new Error("Expected byte part");
}
lengths.push(body.byteLength);
edges.push([body[0]!, body[body.byteLength - 1]!]);
return { ETag: `part-${command.input.PartNumber}` };
}
return {};
});
const data = bytes(Buffer.alloc(PART_BYTES - 1, 1), Buffer.from([2, 3, 4, 5]));
expect(await backend.putObject("large", data, { sizeBytes })).toEqual({
sizeBytes: PART_BYTES + 3,
});
expect(lengths).toEqual([PART_BYTES, 3]);
expect(edges).toEqual([
[1, 2],
[3, 5],
]);
const complete = send.mock.calls.at(-1)?.[0];
expect(complete).toBeInstanceOf(CompleteMultipartUploadCommand);
expect(complete.input).toEqual({
Bucket: "test-bucket",
Key: "archive/team/large",
UploadId: "upload",
IfNoneMatch: "*",
MultipartUpload: {
Parts: [
{ PartNumber: 1, ETag: "part-1" },
{ PartNumber: 2, ETag: "part-2" },
],
},
});
},
);
it("completes an empty unknown-size body through multipart", async () => {
const { send, backend } = fixture();
send
.mockResolvedValueOnce({ UploadId: "upload" })
.mockResolvedValueOnce({ ETag: "empty" })
.mockResolvedValueOnce({});
expect(await backend.putObject("empty", bytes(), {})).toEqual({ sizeBytes: 0 });
expect(send.mock.calls[1]?.[0].input).toMatchObject({ ContentLength: 0, PartNumber: 1 });
expect(send.mock.calls[2]?.[0]).toBeInstanceOf(CompleteMultipartUploadCommand);
});
it.each(["part", "completion", "producer", "size"])(
"aborts multipart when %s fails",
async (failure) => {
const { send, backend } = fixture();
send.mockImplementation(async (command) => {
if (command instanceof CreateMultipartUploadCommand) {
return { UploadId: "upload" };
}
if (command instanceof UploadPartCommand) {
if (failure === "part") {
throw serviceError("AccessDenied", 403);
}
return { ETag: "part" };
}
if (command instanceof CompleteMultipartUploadCommand && failure === "completion") {
throw serviceError("PreconditionFailed", 412);
}
return {};
});
async function* source() {
yield Buffer.from("abc");
if (failure === "producer") {
throw new Error("private-credential-value");
}
}
const result = backend.putObject("broken", source(), {
sizeBytes: failure === "size" ? PART_BYTES + 1 : undefined,
});
await expect(result).rejects.not.toThrow("private-credential-value");
expect(send.mock.calls.at(-1)?.[0]).toBeInstanceOf(AbortMultipartUploadCommand);
expect(send.mock.calls.at(-1)?.[0].input).toEqual({
Bucket: "test-bucket",
Key: "archive/team/broken",
UploadId: "upload",
});
expect(send.mock.calls.at(-1)).toHaveLength(1);
},
);
it("aborts a cancelled multipart upload without passing the cancelled signal to cleanup", async () => {
const { send, backend } = fixture();
const controller = new AbortController();
send.mockImplementation(async (command) => {
if (command instanceof CreateMultipartUploadCommand) {
return { UploadId: "upload" };
}
if (command instanceof UploadPartCommand) {
controller.abort();
throw controller.signal.reason;
}
return {};
});
await expect(
backend.putObject("cancelled", bytes(Buffer.from("abc")), { signal: controller.signal }),
).rejects.toMatchObject({ name: "AbortError" });
expect(send.mock.calls.at(-1)?.[0]).toBeInstanceOf(AbortMultipartUploadCommand);
expect(send.mock.calls.at(-1)).toHaveLength(1);
expect(
send.mock.calls.some(([command]) => command instanceof CompleteMultipartUploadCommand),
).toBe(false);
});
it("cancels a blocked producer and cleans up its upload", async () => {
const { send, backend } = fixture();
const controller = new AbortController();
let release: (() => void) | undefined;
let entered: (() => void) | undefined;
const started = new Promise<void>((resolve) => {
entered = resolve;
});
const waiting = new Promise<void>((resolve) => {
release = resolve;
});
send.mockResolvedValueOnce({ UploadId: "upload" }).mockResolvedValue({});
async function* source() {
entered?.();
await waiting;
yield Buffer.from("late");
}
const result = backend.putObject("blocked", source(), { signal: controller.signal });
const rejected = expect(result).rejects.toMatchObject({ name: "AbortError" });
try {
await started;
controller.abort();
await rejected;
expect(send.mock.calls.at(-1)?.[0]).toBeInstanceOf(AbortMultipartUploadCommand);
} finally {
release?.();
}
});
it("aborts a failed part before waiting for producer cleanup", async () => {
const { send, backend } = fixture();
let release: (() => void) | undefined;
const waiting = new Promise<void>((resolve) => {
release = resolve;
});
send.mockImplementation(async (command) => {
if (command instanceof CreateMultipartUploadCommand) {
return { UploadId: "upload" };
}
if (command instanceof UploadPartCommand) {
throw serviceError("AccessDenied", 403);
}
return {};
});
async function* source() {
try {
yield Buffer.alloc(PART_BYTES);
} finally {
await waiting;
}
}
try {
await expect(backend.putObject("failed-part", source(), {})).rejects.toThrow(
"Object Read & Write",
);
expect(send.mock.calls.at(-1)?.[0]).toBeInstanceOf(AbortMultipartUploadCommand);
} finally {
release?.();
}
});
it("reports cleanup failures without exposing the underlying error", async () => {
const { send, backend } = fixture();
send
.mockResolvedValueOnce({ UploadId: "upload" })
.mockRejectedValue(new Error("private-credential-value"));
await expect(backend.putObject("broken", bytes(Buffer.from("abc")), {})).rejects.toThrow(
"remove incomplete uploads in bucket test-bucket",
);
});
it.each([2, 4])("rejects an incorrect known size %i before publication", async (sizeBytes) => {
const { send, backend } = fixture();
await expect(
backend.putObject("wrong-size", bytes(Buffer.from("abc")), { sizeBytes }),
).rejects.toThrow("declared size");
expect(send).not.toHaveBeenCalled();
});
it.each(["getObject", "statObject"] as const)(
"returns undefined for an absent object in %s",
async (operation) => {
const { send, backend } = fixture();
send.mockRejectedValue(serviceError("NoSuchKey", 404));
expect(await backend[operation]("missing")).toBeUndefined();
send.mockRejectedValue(serviceError("NoSuchBucket", 404));
await expect(backend[operation]("missing")).rejects.toThrow(
`create bucket test-bucket in account ${settings.accountId}`,
);
},
);
it("streams downloads and sanitizes failures while consuming the body", async () => {
const { send, backend } = fixture();
const stream = new ReadableStream<Uint8Array>(
{
start(controller) {
controller.enqueue(Buffer.from("first"));
},
pull(controller) {
controller.error(new Error("private-credential-value"));
},
},
{ highWaterMark: 0 },
);
send.mockResolvedValue({ Body: { transformToWebStream: () => stream } });
const body = await backend.getObject("object");
const iterator = body![Symbol.asyncIterator]();
expect(await iterator.next()).toMatchObject({ value: Buffer.from("first") });
await expect(iterator.next()).rejects.toThrow("R2 download failed");
expect(send.mock.calls[0]?.[0]).toBeInstanceOf(GetObjectCommand);
});
it("paginates within the configured namespace and returns relative keys", async () => {
const { send, backend } = fixture();
send
.mockResolvedValueOnce({
Contents: [{ Key: "archive/team/backups/one", Size: 1, LastModified: new Date(1000) }],
IsTruncated: true,
NextContinuationToken: "next",
})
.mockResolvedValueOnce({ Contents: [{ Key: "archive/team/backups/two", Size: 2 }] });
const objects: StorageObjectInfo[] = [];
for await (const object of backend.listObjects("backups/")) {
objects.push(object);
}
expect(objects).toEqual([
{ key: "backups/one", sizeBytes: 1, modifiedAt: 1000 },
{ key: "backups/two", sizeBytes: 2, modifiedAt: undefined },
]);
expect(send.mock.calls[0]?.[0]).toBeInstanceOf(ListObjectsV2Command);
expect(send.mock.calls[0]?.[0].input).toEqual({
Bucket: "test-bucket",
Prefix: "archive/team/backups/",
ContinuationToken: undefined,
});
expect(send.mock.calls[1]?.[0].input.ContinuationToken).toBe("next");
});
it("stats, probes, deletes, and closes the client", async () => {
const { send, destroy, backend } = fixture();
send
.mockResolvedValueOnce({ ContentLength: 42, LastModified: new Date(1000) })
.mockResolvedValue({});
expect(await backend.statObject("object")).toEqual({
key: "object",
sizeBytes: 42,
modifiedAt: 1000,
});
expect(await backend.probe()).toEqual({});
await backend.deleteObject("object");
await backend.close?.();
expect(send.mock.calls[0]?.[0]).toBeInstanceOf(HeadObjectCommand);
expect(send.mock.calls[1]?.[0]).toBeInstanceOf(HeadBucketCommand);
expect(send.mock.calls[2]?.[0]).toBeInstanceOf(DeleteObjectCommand);
expect(send.mock.calls[2]?.[0].input).toEqual({
Bucket: "test-bucket",
Key: "archive/team/object",
});
expect(destroy).toHaveBeenCalledOnce();
expect(backend.displayTarget).toBe("r2://test-bucket/archive/team");
});
it.each([401, 403, 404, 500])("maps probe HTTP %i into a safe next step", async (code) => {
const { send, backend } = fixture();
send.mockRejectedValue(serviceError("Error", code));
const result = backend.probe();
await expect(result).rejects.toThrow(
code === 401 || code === 403
? "the R2 token needs Object Read & Write on bucket test-bucket"
: code === 404
? `create bucket test-bucket in account ${settings.accountId}`
: "check the bucket settings",
);
await expect(result).rejects.not.toThrow("private-credential-value");
});
});

363
extensions/cloudflare/s3-backend.ts Normal file
View file

@ -0,0 +1,363 @@
import {
AbortMultipartUploadCommand,
CompleteMultipartUploadCommand,
CreateMultipartUploadCommand,
DeleteObjectCommand,
GetObjectCommand,
HeadBucketCommand,
HeadObjectCommand,
ListObjectsV2Command,
PutObjectCommand,
S3ServiceException,
UploadPartCommand,
type CompletedPart,
type S3Client,
} from "@aws-sdk/client-s3";
import type { StorageBackend } from "openclaw/plugin-sdk/plugin-entry";
import { describeR2Target, type R2Settings } from "./settings.js";
const PART_BYTES = 64 * 1024 * 1024;
const MAX_PARTS = 10_000;
class R2StorageError extends Error {}
function missingObject(error: unknown): boolean {
return (
error instanceof S3ServiceException &&
error.name !== "NoSuchBucket" &&
(error.$metadata.httpStatusCode === 404 ||
error.name === "NoSuchKey" ||
error.name === "NotFound")
);
}
function storageError(error: unknown, settings: R2Settings, operation: string): Error {
if (error instanceof R2StorageError) {
return error;
}
if (error instanceof Error && error.name === "AbortError") {
const aborted = new Error("R2 operation aborted; retry when ready.");
aborted.name = "AbortError";
return aborted;
}
if (error instanceof S3ServiceException) {
const code = error.$metadata.httpStatusCode;
if (code === 401 || code === 403) {
return new R2StorageError(
`R2 access denied: the R2 token needs Object Read & Write on bucket ${settings.bucket}.`,
);
}
if (error.name === "NoSuchBucket" || (operation === "probe" && code === 404)) {
return new R2StorageError(
`R2 bucket is missing; create bucket ${settings.bucket} in account ${settings.accountId}.`,
);
}
if (code === 409 || code === 412) {
return new R2StorageError(
"R2 object already exists or a concurrent write conflicted; use a new object key.",
);
}
}
// SDK, stream, and abort-reason errors may contain credentials or signed URLs.
return new R2StorageError(
`R2 ${operation} failed; check the bucket settings, credentials, and connection, then retry.`,
);
}
function assertSize(sizeBytes: number | undefined, actual: number): void {
if (sizeBytes !== undefined && sizeBytes !== actual) {
throw new R2StorageError(
"R2 object does not match its declared size; provide the exact byte count or omit sizeBytes.",
);
}
}
async function* uploadChunks(body: AsyncIterable<Uint8Array>, signal?: AbortSignal) {
signal?.throwIfAborted();
const iterator = body[Symbol.asyncIterator]();
let exhausted = false;
let onAbort: (() => void) | undefined;
const aborted =
signal &&
new Promise<never>((_resolve, reject) => {
onAbort = () => {
const error = new Error("R2 upload aborted; retry when ready.");
error.name = "AbortError";
reject(error);
};
signal.addEventListener("abort", onAbort, { once: true });
});
try {
while (true) {
const next = aborted ? await Promise.race([iterator.next(), aborted]) : await iterator.next();
if (next.done) {
exhausted = true;
return;
}
yield next.value;
}
} finally {
if (onAbort) {
signal?.removeEventListener("abort", onAbort);
}
const closing = iterator.return?.();
if (!exhausted) {
// A producer blocked in next() or finally cannot delay multipart cleanup.
// Observe its eventual close without retaining the upload or leaking a rejection.
void closing?.catch(() => {});
} else {
await closing;
}
}
}
export function createR2Backend(
settings: R2Settings,
client: Pick<S3Client, "send" | "destroy">,
): StorageBackend {
const namespace = settings.prefix ? `${settings.prefix}/` : "";
const keyFor = (key: string) => {
const joined = namespace + key;
if (Buffer.byteLength(joined) > 1024) {
throw new R2StorageError(
"R2 object key exceeds 1024 bytes; shorten the configured prefix or object key.",
);
}
return joined;
};
async function multipart(
Key: string,
body: AsyncIterable<Uint8Array>,
opts: { sizeBytes?: number; signal?: AbortSignal },
): Promise<{ sizeBytes: number }> {
const { UploadId } = await client.send(
new CreateMultipartUploadCommand({ Bucket: settings.bucket, Key }),
{ abortSignal: opts.signal },
);
if (!UploadId) {
throw new R2StorageError(
"R2 did not return an upload ID; check the service status and retry.",
);
}
const upload = { Bucket: settings.bucket, Key, UploadId };
try {
const parts: CompletedPart[] = [];
let sizeBytes = 0;
let used = 0;
// Upload sequentially and reuse one part buffer; no queue can retain extra parts.
const buffer = Buffer.allocUnsafe(PART_BYTES);
const sendPart = async () => {
opts.signal?.throwIfAborted();
if (parts.length === MAX_PARTS) {
throw new R2StorageError(
"R2 upload exceeds 10000 parts; split the object into smaller objects.",
);
}
const PartNumber = parts.length + 1;
const { ETag } = await client.send(
new UploadPartCommand({
...upload,
PartNumber,
Body: buffer.subarray(0, used),
ContentLength: used,
}),
{ abortSignal: opts.signal },
);
if (!ETag) {
throw new R2StorageError("R2 did not acknowledge an uploaded part; retry the upload.");
}
parts.push({ PartNumber, ETag });
used = 0;
};
for await (const chunk of uploadChunks(body, opts.signal)) {
opts.signal?.throwIfAborted();
sizeBytes += chunk.byteLength;
if (opts.sizeBytes !== undefined && sizeBytes > opts.sizeBytes) {
assertSize(opts.sizeBytes, sizeBytes);
}
for (let offset = 0; offset < chunk.byteLength;) {
const count = Math.min(PART_BYTES - used, chunk.byteLength - offset);
buffer.set(chunk.subarray(offset, offset + count), used);
offset += count;
used += count;
if (used === PART_BYTES) {
await sendPart();
}
}
}
assertSize(opts.sizeBytes, sizeBytes);
if (used > 0 || parts.length === 0) {
await sendPart();
}
opts.signal?.throwIfAborted();
// R2 supports conditional multipart publish (release notes, 2023-08-11).
// If-None-Match makes completion atomic; a HeadObject pre-check would race.
await client.send(
new CompleteMultipartUploadCommand({
...upload,
MultipartUpload: { Parts: parts },
IfNoneMatch: "*",
}),
{ abortSignal: opts.signal },
);
return { sizeBytes };
} catch (error) {
try {
// Cleanup must still run when the caller's signal has already aborted.
await client.send(new AbortMultipartUploadCommand(upload));
} catch (cleanupError) {
if (!(cleanupError instanceof S3ServiceException && cleanupError.name === "NoSuchUpload")) {
throw new R2StorageError(
`${storageError(error, settings, "upload").message} Multipart cleanup also failed; remove incomplete uploads in bucket ${settings.bucket}.`,
);
}
}
throw error;
}
}
return {
displayTarget: describeR2Target(settings) ?? `r2://${settings.bucket}`,
async probe(opts) {
try {
await client.send(new HeadBucketCommand({ Bucket: settings.bucket }), {
abortSignal: opts?.signal,
});
return {};
} catch (error) {
throw storageError(error, settings, "probe");
}
},
async putObject(key, body, opts) {
try {
opts.signal?.throwIfAborted();
if (
opts.sizeBytes !== undefined &&
(!Number.isSafeInteger(opts.sizeBytes) || opts.sizeBytes < 0)
) {
throw new R2StorageError("R2 sizeBytes must be a nonnegative safe integer.");
}
const Key = keyFor(key);
if (opts.sizeBytes === undefined || opts.sizeBytes > PART_BYTES) {
return await multipart(Key, body, opts);
}
const buffer = Buffer.allocUnsafe(opts.sizeBytes);
let sizeBytes = 0;
for await (const chunk of uploadChunks(body, opts.signal)) {
opts.signal?.throwIfAborted();
if (sizeBytes + chunk.byteLength > buffer.length) {
assertSize(opts.sizeBytes, sizeBytes + chunk.byteLength);
}
buffer.set(chunk, sizeBytes);
sizeBytes += chunk.byteLength;
}
assertSize(opts.sizeBytes, sizeBytes);
opts.signal?.throwIfAborted();
await client.send(
new PutObjectCommand({
Bucket: settings.bucket,
Key,
Body: buffer,
ContentLength: sizeBytes,
IfNoneMatch: "*",
}),
{ abortSignal: opts.signal },
);
return { sizeBytes };
} catch (error) {
throw storageError(error, settings, "upload");
}
},
async getObject(key, opts) {
try {
const { Body } = await client.send(
new GetObjectCommand({ Bucket: settings.bucket, Key: keyFor(key) }),
{ abortSignal: opts?.signal },
);
if (!Body) {
throw new R2StorageError("R2 returned no object body; retry the download.");
}
const stream = Body.transformToWebStream();
return (async function* () {
try {
for await (const chunk of stream) {
opts?.signal?.throwIfAborted();
yield chunk;
}
} catch (error) {
throw storageError(error, settings, "download");
}
})();
} catch (error) {
if (missingObject(error)) {
return undefined;
}
throw storageError(error, settings, "download");
}
},
async statObject(key, opts) {
try {
const result = await client.send(
new HeadObjectCommand({ Bucket: settings.bucket, Key: keyFor(key) }),
{ abortSignal: opts?.signal },
);
if (result.ContentLength === undefined) {
throw new R2StorageError("R2 returned no object size; retry the metadata request.");
}
return { key, sizeBytes: result.ContentLength, modifiedAt: result.LastModified?.getTime() };
} catch (error) {
if (missingObject(error)) {
return undefined;
}
throw storageError(error, settings, "stat");
}
},
async *listObjects(prefix, opts) {
try {
const Prefix = keyFor(prefix);
let ContinuationToken: string | undefined;
do {
opts?.signal?.throwIfAborted();
const page = await client.send(
new ListObjectsV2Command({ Bucket: settings.bucket, Prefix, ContinuationToken }),
{ abortSignal: opts?.signal },
);
for (const object of page.Contents ?? []) {
if (
object.Key === undefined ||
object.Size === undefined ||
!object.Key.startsWith(Prefix)
) {
throw new R2StorageError("R2 returned invalid object metadata; retry the listing.");
}
yield {
key: object.Key.slice(namespace.length),
sizeBytes: object.Size,
modifiedAt: object.LastModified?.getTime(),
};
}
const next = page.IsTruncated ? page.NextContinuationToken : undefined;
if (page.IsTruncated && (!next || next === ContinuationToken)) {
throw new R2StorageError("R2 did not advance its listing cursor; retry the listing.");
}
ContinuationToken = next;
} while (ContinuationToken);
} catch (error) {
throw storageError(error, settings, "list");
}
},
async deleteObject(key, opts) {
try {
await client.send(new DeleteObjectCommand({ Bucket: settings.bucket, Key: keyFor(key) }), {
abortSignal: opts?.signal,
});
} catch (error) {
throw storageError(error, settings, "delete");
}
},
async close() {
client.destroy();
},
};
}

128
extensions/cloudflare/settings.test.ts Normal file
View file

@ -0,0 +1,128 @@
import type { StorageProvider } from "openclaw/plugin-sdk/plugin-entry";
import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api";
import { describe, expect, it, vi } from "vitest";
import { r2StorageProvider } from "./api.js";
import plugin from "./index.js";
import { r2Endpoint, type R2Settings } from "./settings.js";
const settings = {
accountId: "0123456789abcdef0123456789abcdef",
bucket: "openclaw-artifacts",
accessKeyId: { source: "env", provider: "default", id: "R2_ACCESS_KEY_ID" },
secretAccessKey: { source: "env", provider: "default", id: "R2_SECRET_ACCESS_KEY" },
} satisfies R2Settings;
describe("Cloudflare R2 provider settings", () => {
it("registers and opens its storage provider through the plugin entrypoint", async () => {
const providers: StorageProvider[] = [];
plugin.register?.(
createTestPluginApi({
registerStorageProvider(provider) {
providers.push(provider);
},
}),
);
expect(providers).toHaveLength(1);
const provider = providers[0]!;
expect(provider.id).toBe("r2");
const resolveSecret = vi.fn(async () => "example-credential-not-real");
const sessionToken = { source: "env", provider: "default", id: "R2_SESSION_TOKEN" };
const backend = await provider.open({
locationName: "offsite",
settings: { ...settings, sessionToken },
resolveSecret,
});
try {
expect(backend.displayTarget).toBe("r2://openclaw-artifacts");
expect(resolveSecret.mock.calls).toEqual([
[settings.accessKeyId],
[settings.secretAccessKey],
[sessionToken],
]);
} finally {
await backend.close?.();
}
});
it.each([
{},
{ bucket: "a-0" },
{ bucket: "a".repeat(63) },
{ prefix: "team/Archive_2026-09.30" },
{ prefix: "a".repeat(512) },
{ jurisdiction: "eu" },
{ jurisdiction: "fedramp" },
{ sessionToken: { source: "env", provider: "default", id: "R2_SESSION_TOKEN" } },
])("accepts supported settings %j", (overrides) => {
expect(r2StorageProvider.validateSettings?.({ ...settings, ...overrides })).toBeUndefined();
});
it.each([
["accountId", "0123456789ABCDEF0123456789ABCDEF"],
["accountId", "0123456789abcdef"],
["accountId", "g".repeat(32)],
["bucket", "ab"],
["bucket", "a".repeat(64)],
["bucket", "Uppercase"],
["bucket", "bucket.name"],
["bucket", "-bucket"],
["bucket", "bucket-"],
["prefix", ""],
["prefix", "/archive"],
["prefix", "archive/"],
["prefix", "archive//daily"],
["prefix", "archive/./daily"],
["prefix", "archive/../daily"],
["prefix", "archive\\daily"],
["prefix", "archive/with space"],
["prefix", "a".repeat(513)],
["jurisdiction", "us"],
["jurisdiction", "EU"],
])("rejects invalid %s value %j", (key, value) => {
expect(r2StorageProvider.validateSettings?.({ ...settings, [key]: value })).toContain(key);
});
it.each(["accessKeyId", "secretAccessKey", "sessionToken"])(
"requires a valid SecretRef for %s without disclosing plaintext",
(key) => {
const plaintext = "example-r2-credential-not-real";
const error = r2StorageProvider.validateSettings?.({ ...settings, [key]: plaintext });
expect(error).toContain(`${key} must be a valid SecretRef`);
expect(error).not.toContain(plaintext);
expect(
r2StorageProvider.validateSettings?.({
...settings,
[key]: { source: "env", provider: "default", id: "invalid-env-name" },
}),
).toContain(`${key} must be a valid SecretRef`);
},
);
it.each(["accessKeyId", "secretAccessKey"])("requires %s", (key) => {
expect(r2StorageProvider.validateSettings?.({ ...settings, [key]: undefined })).toContain(key);
});
it("rejects unsupported settings rather than silently using a different endpoint", () => {
expect(
r2StorageProvider.validateSettings?.({ ...settings, endpoint: "https://example.com" }),
).toContain("settings only accept");
});
it.each([
[undefined, "https://0123456789abcdef0123456789abcdef.r2.cloudflarestorage.com"],
["eu", "https://0123456789abcdef0123456789abcdef.eu.r2.cloudflarestorage.com"],
["fedramp", "https://0123456789abcdef0123456789abcdef.fedramp.r2.cloudflarestorage.com"],
] as const)("routes jurisdiction %s to its R2 endpoint", (jurisdiction, endpoint) => {
expect(r2Endpoint({ ...settings, jurisdiction })).toBe(endpoint);
});
it("describes bucket and prefix synchronously without credentials", () => {
expect(r2StorageProvider.describeTarget?.({ bucket: "openclaw-artifacts" })).toBe(
"r2://openclaw-artifacts",
);
expect(
r2StorageProvider.describeTarget?.({ bucket: "openclaw-artifacts", prefix: "team/archive" }),
).toBe("r2://openclaw-artifacts/team/archive");
expect(r2StorageProvider.describeTarget?.({ bucket: "invalid.bucket" })).toBeUndefined();
});
});

107
extensions/cloudflare/settings.ts Normal file
View file

@ -0,0 +1,107 @@
import { isSecretRef, isValidSecretRef, type SecretInput } from "openclaw/plugin-sdk/secret-input";
export type R2Settings = {
accountId: string;
bucket: string;
prefix?: string;
jurisdiction?: "eu" | "fedramp";
accessKeyId: SecretInput;
secretAccessKey: SecretInput;
sessionToken?: SecretInput;
};
class R2SettingsError extends Error {}
function validBucket(value: unknown): value is string {
return typeof value === "string" && /^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/.test(value);
}
function validPrefix(value: unknown): value is string | undefined {
return (
value === undefined ||
(typeof value === "string" &&
Buffer.byteLength(value) <= 512 &&
value
.split("/")
.every((part) => /^[A-Za-z0-9._-]+$/.test(part) && part !== "." && part !== ".."))
);
}
function credential(value: unknown, key: string) {
if (!isSecretRef(value) || !isValidSecretRef(value)) {
throw new R2SettingsError(
`R2 ${key} must be a valid SecretRef; store the credential in a secret provider first.`,
);
}
return value;
}
export function parseR2Settings(settings: Readonly<Record<string, unknown>>): R2Settings {
const { accountId, bucket, prefix, jurisdiction } = settings;
if (typeof accountId !== "string" || !/^[a-f0-9]{32}$/.test(accountId)) {
throw new R2SettingsError("R2 accountId must be 32 lowercase hexadecimal characters.");
}
if (!validBucket(bucket)) {
throw new R2SettingsError(
"R2 bucket must be 3–63 lowercase letters, digits, or hyphens, beginning and ending with a letter or digit.",
);
}
if (!validPrefix(prefix)) {
throw new R2SettingsError(
"R2 prefix must contain key-safe segments without leading or trailing /, . or .. segments, and be at most 512 bytes.",
);
}
if (jurisdiction !== undefined && jurisdiction !== "eu" && jurisdiction !== "fedramp") {
throw new R2SettingsError("R2 jurisdiction must be eu or fedramp, or omitted.");
}
const allowed = new Set([
"accountId",
"bucket",
"prefix",
"jurisdiction",
"accessKeyId",
"secretAccessKey",
"sessionToken",
]);
if (Object.keys(settings).some((key) => !allowed.has(key))) {
throw new R2SettingsError(
"R2 settings only accept accountId, bucket, prefix, jurisdiction, accessKeyId, secretAccessKey, and sessionToken.",
);
}
return {
accountId,
bucket,
prefix,
jurisdiction,
accessKeyId: credential(settings.accessKeyId, "accessKeyId"),
secretAccessKey: credential(settings.secretAccessKey, "secretAccessKey"),
sessionToken:
settings.sessionToken === undefined
? undefined
: credential(settings.sessionToken, "sessionToken"),
};
}
export function validateR2Settings(
settings: Readonly<Record<string, unknown>>,
): string | undefined {
try {
parseR2Settings(settings);
return undefined;
} catch (error) {
return error instanceof R2SettingsError
? error.message
: "R2 settings could not be read; check the location configuration.";
}
}
export function describeR2Target(settings: Readonly<Record<string, unknown>>): string | undefined {
if (!validBucket(settings.bucket) || !validPrefix(settings.prefix)) {
return undefined;
}
return `r2://${settings.bucket}${settings.prefix ? `/${settings.prefix}` : ""}`;
}
export function r2Endpoint(settings: R2Settings): string {
return `https://${settings.accountId}${settings.jurisdiction ? `.${settings.jurisdiction}` : ""}.r2.cloudflarestorage.com`;
}

3
extensions/cloudflare/tsconfig.json Normal file
View file

@ -0,0 +1,3 @@
{
"extends": "../tsconfig.package-boundary.base.json"
}

5
packages/gateway-protocol/src/core-request-params.ts
View file

@ -1,5 +1,6 @@
import type { Static } from "typebox";
import type * as AgentSchema from "./schema/agent.js";
import type { BackupStatusParams } from "./schema/backup.js";
import type * as BoardSchema from "./schema/board.js";
import type { CanvasDocumentPreviewParams, CanvasDocumentViewParams } from "./schema/canvas.js";
import type { CommandsListParams } from "./schema/commands.js";
@ -15,6 +16,7 @@ import type { LogsTailParams } from "./schema/logs-chat.js";
import type * as PortalSchema from "./schema/portals.js";
import type { PresenceActivityParams, PresenceQueryParams } from "./schema/presence.js";
import type * as GitHubSchema from "./schema/session-github-publication.js";
import type { StorageLocationsListParams, StorageLocationsProbeParams } from "./schema/storage.js";
import type {
ThemesListParams,
ThemesGetParams,
@ -27,6 +29,9 @@ import type * as UsersSchema from "./schema/users.js";
/** Schema-derived payload ownership for statically validated core Gateway methods. */
export type GatewayCoreRequestParams = {
"backup.status": BackupStatusParams;
"storage.locations.list": StorageLocationsListParams;
"storage.locations.probe": StorageLocationsProbeParams;
"presence.activity": PresenceActivityParams;
"cron.history": CronHistoryParams;
"users.personalFile.get": UsersSchema.UsersPersonalFileGetParams;

2
packages/gateway-protocol/src/index.ts
View file

@ -63,6 +63,8 @@ export {
} from "./schema/sessions-create.js";
export * from "./schema/projects.js";
export * from "./migration-api.js";
export * from "./schema/storage.js";
export * from "./schema/backup.js";
export * from "./restart-unavailable.js";
export type * from "./public-session-catalog.js";
export * from "./validator-registry.js";

2
packages/gateway-protocol/src/schema-modules.ts
View file

@ -52,6 +52,8 @@ export * from "./schema/sessions-reactions.js";
export * from "./schema/sessions-catalog.js";
export * from "./schema/skill-history.js";
export * from "./schema/snapshot.js";
export * from "./schema/storage.js";
export * from "./schema/backup.js";
export * from "./schema/system-info.js";
export * from "./schema/system-event.js";
export * from "./schema/task-suggestions.js";

65
packages/gateway-protocol/src/schema/backup.ts Normal file
View file

@ -0,0 +1,65 @@
import type { Static } from "typebox";
import { Type } from "typebox";
import { closedObject } from "./closed-object.js";
import { StorageLocationsListResultSchema } from "./storage.js";
const BackupKindSchema = Type.Union([
Type.Literal("archive"),
Type.Literal("sqlite-snapshot"),
Type.Literal("git"),
Type.Literal("external"),
]);
const BytesSchema = Type.Integer({ minimum: 0 });
export const BackupRunLocationSchema = closedObject({
name: Type.String(),
provider: Type.String(),
locationId: Type.String(),
key: Type.String(),
namespace: Type.String(),
plaintextBytes: BytesSchema,
storedBytes: BytesSchema,
});
export const BackupRunRetentionSchema = closedObject({ kept: BytesSchema, deleted: BytesSchema });
export const BackupRunRecordSchema = closedObject({
id: Type.String(),
createdAt: Type.Number(),
archivePath: Type.String(),
status: Type.Union([Type.Literal("ok"), Type.Literal("failed")]),
kind: BackupKindSchema,
target: Type.Optional(Type.String()),
namespace: Type.Optional(Type.String()),
error: Type.Optional(Type.String()),
pushFailed: Type.Optional(Type.Literal(true)),
bytes: Type.Optional(BytesSchema),
location: Type.Optional(BackupRunLocationSchema),
retention: Type.Optional(BackupRunRetentionSchema),
});
export const BackupStatusParamsSchema = closedObject({});
export const BackupStatusResultSchema = closedObject({
targets: Type.Array(
closedObject({
kind: BackupKindSchema,
target: Type.String(),
namespace: Type.Optional(Type.String()),
latest: BackupRunRecordSchema,
latestOk: Type.Optional(BackupRunRecordSchema),
}),
),
schedules: Type.Array(
closedObject({
id: Type.String(),
mode: Type.Union([Type.Literal("git"), Type.Literal("offsite")]),
target: Type.String(),
namespace: Type.Optional(Type.String()),
enabled: Type.Boolean(),
everyMs: Type.Number(),
nextRunAtMs: Type.Optional(Type.Number()),
}),
),
locations: StorageLocationsListResultSchema.properties.locations,
});
export type BackupRunRecord = Static<typeof BackupRunRecordSchema>;
export type BackupRunLocation = Static<typeof BackupRunLocationSchema>;
export type BackupRunRetention = Static<typeof BackupRunRetentionSchema>;
export type BackupStatusParams = Static<typeof BackupStatusParamsSchema>;
export type BackupStatusResult = Static<typeof BackupStatusResultSchema>;

39
packages/gateway-protocol/src/schema/storage.ts Normal file
View file

@ -0,0 +1,39 @@
import type { Static } from "typebox";
import { Type } from "typebox";
import { closedObject } from "./closed-object.js";
import { NonEmptyString } from "./primitives.js";
const StorageLocationNameSchema = Type.String({ pattern: "^[a-z0-9][a-z0-9-]{0,62}$" });
export const StorageLocationsListParamsSchema = closedObject({});
export const StorageLocationsListResultSchema = closedObject({
locations: Type.Array(
closedObject({
name: StorageLocationNameSchema,
provider: NonEmptyString,
displayTarget: Type.Optional(NonEmptyString),
encrypted: Type.Boolean(),
}),
),
});
export const StorageLocationsProbeParamsSchema = closedObject({
name: StorageLocationNameSchema,
});
export const StorageLocationsProbeResultSchema = closedObject({
state: Type.Union([
Type.Literal("ok"),
Type.Literal("unavailable"),
Type.Literal("uninitialized"),
Type.Literal("wrong-key"),
Type.Literal("error"),
]),
freeBytes: Type.Optional(Type.Number({ minimum: 0 })),
totalBytes: Type.Optional(Type.Number({ minimum: 0 })),
message: Type.Optional(Type.String()),
});
export type StorageLocationsListParams = Static<typeof StorageLocationsListParamsSchema>;
export type StorageLocationsListResult = Static<typeof StorageLocationsListResultSchema>;
export type StorageLocationsProbeParams = Static<typeof StorageLocationsProbeParamsSchema>;
export type StorageLocationsProbeResult = Static<typeof StorageLocationsProbeResultSchema>;

3
packages/gateway-protocol/src/validator-registry.ts
View file

@ -25,6 +25,9 @@ export {
// Validator names mirror schemas so callers can pair them with wire contracts.
export const validateCommandsListParams = compile(S.CommandsListParamsSchema);
export const validateBackupStatusParams = compile(S.BackupStatusParamsSchema);
export const validateStorageLocationsListParams = compile(S.StorageLocationsListParamsSchema);
export const validateStorageLocationsProbeParams = compile(S.StorageLocationsProbeParamsSchema);
export const validateComputerStatusParams = compile(S.ComputerStatusParamsSchema);
export const validateComputerInvokeParams = compile(S.ComputerInvokeParamsSchema);
export const validateCanvasDocumentPreviewParams = compile(S.CanvasDocumentPreviewParamsSchema);

10
pnpm-lock.yaml generated
View file

@ -904,6 +904,16 @@ importers:
specifier: workspace:*
version: link:../..
extensions/cloudflare:
dependencies:
'@aws-sdk/client-s3':
specifier: 3.1136.0
version: 3.1136.0
devDependencies:
'@openclaw/plugin-sdk':
specifier: workspace:*
version: link:../../packages/plugin-sdk
extensions/cloudflare-ai-gateway:
devDependencies:
'@openclaw/plugin-sdk':

4
scripts/check-protocol-registry.mts
View file

@ -155,8 +155,8 @@ const ownerModules = [
...schemaModulesSource.matchAll(/^export \* from "\.\/schema\/([^"]+)\.js";$/gmu),
].map(([, moduleName = ""]) => moduleName);
check(
ownerModules.length === 70 && new Set(ownerModules).size === ownerModules.length,
"schema-modules.ts must contain one unique 70-module owner list",
ownerModules.length === 72 && new Set(ownerModules).size === ownerModules.length,
"schema-modules.ts must contain one unique 72-module owner list",
);
check(
schemaModulesSource.split("\n").filter(Boolean).length === ownerModules.length,

196
scripts/e2e/lib/upgrade-survivor/backup-schedule.mjs Normal file
View file

@ -0,0 +1,196 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { DatabaseSync } from "node:sqlite";
const root = process.env.OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT;
const artifacts = process.env.OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT;
const stateDir = process.env.OPENCLAW_STATE_DIR;
const configPath = process.env.OPENCLAW_CONFIG_PATH;
assert(root && artifacts && stateDir && configPath, "Missing isolated survivor paths");
const repository = path.join(root, "git-backups");
const archive = path.join(root, "archive-backup.tar.gz");
const beforePath = path.join(artifacts, "backup-schedule-before.json");
const declarationKey = "openclaw-backup-scheduled";
function readJson(file) {
return JSON.parse(fs.readFileSync(file, "utf8"));
}
function writeJson(file, value) {
fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
}
function cli(label, args, json = false) {
const started = Date.now();
const result = spawnSync("openclaw", args, {
encoding: "utf8",
timeout: 900_000,
maxBuffer: 16 * 1024 * 1024,
});
fs.writeFileSync(path.join(artifacts, `${label}.out`), result.stdout ?? "");
fs.writeFileSync(path.join(artifacts, `${label}.err`), result.stderr ?? "");
console.log(`${label}: exit=${result.status} durationMs=${Date.now() - started}`);
assert.equal(result.status, 0, `${label} failed: ${result.stderr}\n${result.stdout}`);
return json ? JSON.parse(result.stdout) : result.stdout;
}
function rpc(label, method, params = {}) {
return cli(
label,
["gateway", "call", method, "--params", JSON.stringify(params), "--json"],
true,
);
}
function schedule(label) {
const result = cli(label, ["cron", "list", "--all", "--json"], true);
const jobs = result.jobs.filter((job) => job.declarationKey === declarationKey);
assert.equal(jobs.length, 1, "Expected exactly one Gateway-owned Git backup schedule");
const job = jobs[0];
assert.equal(job.enabled, true);
assert.deepEqual(job.schedule.kind, "every");
assert.equal(job.schedule.everyMs, 86_400_000);
assert.equal(job.payload.kind, "command");
assert.deepEqual(job.payload.argv, [
"openclaw",
"backup",
"git",
"create",
"--repository",
repository,
"--all",
]);
return { id: job.id, declarationKey: job.declarationKey, argv: job.payload.argv };
}
function ledgerRows() {
const db = new DatabaseSync(path.join(stateDir, "state", "openclaw.sqlite"), { readOnly: true });
try {
return db
.prepare(
"SELECT id, created_at, archive_path, status, manifest_json FROM backup_runs ORDER BY created_at, id",
)
.all()
.map((row) => Object.assign({}, row));
} finally {
db.close();
}
}
function doctorErrors(label) {
const report = cli(label, ["doctor", "--lint", "--json", "--severity-min", "error"], true);
assert(report.checksRun > 0, "Doctor ran no checks");
return report.findings.filter((finding) => finding.severity === "error");
}
function assertNoStorageConfig() {
assert.equal(
readJson(configPath).storage?.locations,
undefined,
"Upgrade wrote storage.locations",
);
}
const mode = process.argv[2];
const packageRoot = process.argv[3];
if (mode === "configure") {
const workspace = path.join(root, "workspace");
fs.mkdirSync(workspace, { recursive: true });
fs.writeFileSync(path.join(workspace, "MEMORY.md"), "Preserve this existing backup schedule.\n");
for (const [key, value] of Object.entries({
"gateway.mode": "local",
"gateway.auth": { mode: "token", token: process.env.GATEWAY_AUTH_TOKEN_REF },
"agents.defaults.workspace": workspace,
})) {
cli(`backup-config-${key}`, ["config", "set", key, JSON.stringify(value), "--strict-json"]);
}
cli("backup-baseline-doctor", ["doctor", "--fix", "--non-interactive"]);
cli("backup-git-init", ["backup", "git", "init", "--repository", repository, "--json"], true);
assertNoStorageConfig();
} else if (mode === "seed") {
cli("backup-enable", ["backup", "enable", "--repository", repository, "--every", "24h"]);
const original = schedule("backup-cron-before");
cli(
"backup-git-create",
["backup", "git", "create", "--repository", repository, "--all", "--json"],
true,
);
cli("backup-archive-create", ["backup", "create", "--output", archive, "--json"], true);
const rows = ledgerRows().filter(
(row) => row.archive_path === repository || row.archive_path === archive,
);
assert.equal(rows.length, 2, "Baseline must record both Git and archive outcomes");
assert(
rows.every((row) => row.status === "ok"),
"Baseline backups must succeed",
);
const errors = doctorErrors("backup-doctor-before");
assertNoStorageConfig();
const baseline = readJson(path.join(packageRoot, "dist", "build-info.json"));
assert.equal(baseline.version, "2026.9.7");
writeJson(beforePath, { baseline, schedule: original, rows, errors });
console.log(
`Seeded Git schedule ${original.declarationKey} and ${rows.length} successful ledger rows.`,
);
} else if (mode === "assert") {
const before = readJson(beforePath);
assert.deepEqual(
schedule("backup-cron-after"),
before.schedule,
"Upgrade changed schedule identity, declaration, or argv",
);
const status = rpc("backup-status-after", "backup.status");
const active = status.schedules.find((entry) => entry.id === before.schedule.id);
assert(active, "backup.status omitted the existing schedule");
assert.equal(active.mode, "git");
assert.equal(active.target, repository);
assert.equal(active.enabled, true);
assert.equal(active.everyMs, 86_400_000);
for (const row of before.rows) {
const target = status.targets.find((entry) => entry.latest.id === row.id);
assert(target, `backup.status omitted pre-update row ${row.id}`);
assert.equal(target.kind, row.archive_path === repository ? "git" : "archive");
assert.equal(target.latest.archivePath, row.archive_path);
assert.equal(target.latest.createdAt, row.created_at);
assert.equal(target.latest.status, "ok");
assert.equal(target.latestOk.id, row.id);
}
const rows = ledgerRows();
for (const row of before.rows) {
assert.deepEqual(
rows.find((entry) => entry.id === row.id),
row,
"Upgrade rewrote a pre-update ledger row",
);
}
const overview = cli("backup-overview-after", ["status"]);
assert.match(overview, /Backups[^\n]*last ok/u, "openclaw status omitted successful backup line");
const errors = doctorErrors("backup-doctor-after");
assert.deepEqual(errors, before.errors, "Doctor introduced new errors");
assertNoStorageConfig();
assert.deepEqual(status.locations, [], "backup.status invented storage locations");
const plugins = rpc("backup-plugins-after", "plugins.list");
const cloudflare = plugins.plugins.find((entry) => entry.id === "cloudflare");
assert(cloudflare, "Plugin inventory omitted bundled Cloudflare");
assert(
["disabled", "unloaded"].includes(cloudflare.runtime?.state),
"Cloudflare activated without an R2 location",
);
assert.equal(cloudflare.enabled, false, "Cloudflare became enabled");
const result = {
baseline: before.baseline,
candidate: readJson(path.join(packageRoot, "dist", "build-info.json")),
schedule: { ...before.schedule, mode: active.mode, preserved: true },
ledgerRows: before.rows.map((row) => ({ id: row.id, preserved: true, visibleInStatus: true })),
statusLine: overview.split("\n").find((line) => /Backups/u.test(line)),
doctor: { baselineErrors: before.errors.length, candidateErrors: errors.length, newErrors: 0 },
storageLocationsWritten: false,
cloudflare: { enabled: cloudflare.enabled, runtime: cloudflare.runtime.state },
};
writeJson(path.join(artifacts, "backup-schedule.json"), result);
console.log(JSON.stringify(result, null, 2));
} else {
throw new Error("Expected configure, seed, or assert");
}

29
scripts/e2e/lib/upgrade-survivor/run.sh
View file

@ -355,6 +355,9 @@ const summary = {
backupRollback: process.env.SUMMARY_SCENARIO === "legacy-operator-state"
? readJsonOrNull(process.env.SUMMARY_BACKUP_ROLLBACK)
: undefined,
backupSchedule: process.env.SUMMARY_SCENARIO === "backup-schedule"
? readJsonOrNull(path.join(path.dirname(process.env.SUMMARY_JSON), "backup-schedule.json"))
: undefined,
nativeAssignmentEligibility: readJsonOrNull(path.join(path.dirname(process.env.SUMMARY_JSON), "native-assignment-eligibility.json")),
nativeAssignments: process.env.SUMMARY_SCENARIO === "legacy-operator-state"
? readJsonOrNull(path.join(path.dirname(process.env.SUMMARY_JSON), "native-assignment-proof.json"))
@ -2258,6 +2261,32 @@ phase validate-worker-cell validate_worker_cell
phase reset-run-state reset_run_state
phase install-baseline install_baseline
phase initialize-state initialize_state
if [ "$SCENARIO" = "backup-schedule" ]; then
if [ "$baseline_spec" != "openclaw@2026.9.7" ] || [ "$CANDIDATE_KIND" != "tarball" ] ||
[ "$UPDATE_RESTART_MODE" != "manual" ] || [ "$ROOT_MANAGED_VPS" != "0" ] || [ "$LIVE_ENABLED" != "0" ]; then
echo "backup-schedule requires published openclaw@2026.9.7, a candidate tarball, isolated manual restart, and no live provider" >&2
exit 2
fi
phase configure-backup-baseline node scripts/e2e/lib/upgrade-survivor/backup-schedule.mjs configure
phase start-backup-baseline start_gateway
phase seed-backup-schedule node scripts/e2e/lib/upgrade-survivor/backup-schedule.mjs seed "$(package_root)"
phase stop-backup-baseline stop_gateway
phase resolve-backup-candidate resolve_candidate_version
phase backup-candidate-package-identity node scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs \
candidate "$(package_root)" "$CANDIDATE_SPEC"
phase update-backup-candidate update_candidate
phase backup-installed-package-identity node scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs \
installed "$(package_root)" "$CANDIDATE_SPEC"
phase backup-doctor run_doctor
phase validate-backup-post-doctor-config validate_post_doctor_config
phase start-backup-candidate start_gateway
phase backup-gateway-probes check_gateway_probes
phase backup-gateway-status check_gateway_status
phase assert-backup-schedule node scripts/e2e/lib/upgrade-survivor/backup-schedule.mjs assert "$(package_root)"
run_completed="1"
echo "Backup schedule survivor passed: published ${baseline_version} updater preserved Git declaration, argv, ledger, and health without enabling storage."
exit 0
fi
if [ "$SCENARIO" = "dreaming-cron-doctor" ]; then
if [ "$baseline_spec" != "openclaw@2026.9.6" ] || [ "$CANDIDATE_KIND" != "tarball" ] ||
[ "$UPDATE_RESTART_MODE" != "manual" ] || [ "$ROOT_MANAGED_VPS" != "0" ] || [ "$LIVE_ENABLED" != "0" ]; then

1
scripts/generate-plugin-inventory-doc.mts
View file

@ -270,6 +270,7 @@ function resolveDescription({ manifest, packageJson }: PluginSourceEntry) {
realtimeTranscriptionProviders: "Adds realtime transcription provider support.",
realtimeVoiceProviders: "Adds realtime voice provider support.",
speechProviders: "Adds text-to-speech provider support.",
storageProviders: "Adds storage location transport support.",
tools: "Adds agent-callable tools.",
videoGenerationProviders: "Adds video generation provider support.",
webContentExtractors: "Adds readable web content extraction.",

1
scripts/lib/bundled-runtime-sidecar-paths.json
View file

@ -2,6 +2,7 @@
"dist/extensions/a2a/runtime-api.js",
"dist/extensions/browser/runtime-api.js",
"dist/extensions/canvas/runtime-api.js",
"dist/extensions/cloudflare/runtime-api.js",
"dist/extensions/copilot-proxy/runtime-api.js",
"dist/extensions/google/runtime-api.js",
"dist/extensions/lmstudio/runtime-api.js",

1
scripts/lib/docker-e2e-plan.mts
View file

@ -805,6 +805,7 @@ export function requiredPrepublishPluginPackagesForLanes(
if (
!scenario ||
scenario === "abandoned-update" ||
scenario === "backup-schedule" ||
scenario === "custom-plugin-siblings" ||
scenario === "projects-doctor" ||
scenario === "channel-owner-policy" ||

4
scripts/lib/upgrade-survivor-policy.mjs
View file

@ -54,6 +54,7 @@ export function isTrustedHarnessOwnedUpgradeSurvivorScenario(scenario) {
const aggregateScenarios = UPGRADE_SURVIVOR_SCENARIOS.filter(
(scenario) =>
scenario !== "abandoned-update" &&
scenario !== "backup-schedule" &&
scenario !== "missing-configured-plugin-migration" &&
scenario !== "missing-load-path" &&
scenario !== "projects-doctor" &&
@ -172,6 +173,9 @@ function comparePublishedReleaseVersion(a, b) {
}
export function supportsUpgradeSurvivorScenarioAtBaseline(scenario, baselineSpec) {
if (scenario === "backup-schedule") {
return baselineSpec === "openclaw@2026.9.7";
}
if (scenario === "missing-load-path") {
const release = parseReleaseVersion((baselineSpec ?? "").replace(/^openclaw@/u, ""));
// Floating tags are checked again against the installed baseline before seeding.

1
scripts/lib/upgrade-survivor-scenarios.json
View file

@ -1,6 +1,7 @@
{
"scenarios": [
"base",
"backup-schedule",
"abandoned-update",
"legacy-operator-state",
"workshop-doctor-recovery",

5
scripts/plugin-sdk-surface-report.mts
View file

@ -193,11 +193,10 @@ export function readPluginSdkSurfaceBudgets(env: NodeJS.ProcessEnv = process.env
158,
env,
),
// #160931 (b4ae783fbfd) added three callable agent-harness-runtime exports
// without its ratchet update; these pin exactly that growth.
// Includes four approved storage transport contract types on plugin-entry.
publicExports: readPluginSdkSurfaceBudgetEnv(
"OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS",
4587,
4591,
env,
),
publicFunctionExports: readPluginSdkSurfaceBudgetEnv(

2
scripts/pr-lib/wrapper-components.txt
View file

@ -482,6 +482,7 @@ src/config/plugin-auto-enable.prefer-over.ts
src/config/plugin-install-record-map.ts
src/config/plugins-allowlist.ts
src/config/provider-policy.ts
src/config/provider-settings.ts
src/config/recovery-policy.ts
src/config/redact-argv.ts
src/config/redact-sentinel.ts
@ -563,6 +564,7 @@ src/config/zod-schema.secret-input.ts
src/config/zod-schema.sensitive.ts
src/config/zod-schema.session-config.ts
src/config/zod-schema.session.ts
src/config/zod-schema.storage.ts
src/config/zod-schema.telemetry.ts
src/config/zod-schema.ts
src/cron/completion-status.ts

4
src/cli/program/command-registry-core.ts
View file

@ -43,6 +43,10 @@ const coreEntrySpecs: readonly CommandGroupDescriptorSpec<[ctx: ProgramContext]>
["migrate"],
async (program) => (await import("./register.migrate.js")).registerMigrateCommand(program),
],
[
["storage"],
async (program) => (await import("./register.storage.js")).registerStorageCommand(program),
],
[
["audit"],
async (program) => (await import("./register.audit.js")).registerAuditCommand(program),

7
src/cli/program/core-command-descriptors.ts
View file

@ -56,6 +56,13 @@ export const CORE_CLI_COMMAND_DESCRIPTORS = [
description: "Import state from another agent system",
hasSubcommands: true,
},
{
name: "storage",
description: "List, initialize, and test configured storage locations",
hasSubcommands: true,
parentDefaultHelp: true,
machineOutput: ({ argv }) => hasMachineOutputOption(argv, "--json"),
},
{
name: "doctor",
description: "Health checks + quick fixes for the gateway and channels",

484
src/cli/program/register.backup.offsite.test.ts Normal file
View file

@ -0,0 +1,484 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { json } from "node:stream/consumers";
import { Command } from "commander";
import { afterEach, describe, expect, it, vi } from "vitest";
import { claimBackupNamespace } from "../../commands/backup-namespace.js";
import type { OffsiteBackupResult } from "../../commands/backup-remote.js";
import { getRuntimeConfig } from "../../config/config.js";
import * as deviceIdentity from "../../infra/device-identity-async.js";
import { loadDeviceIdentityIfPresentAsync } from "../../infra/device-identity-async.js";
import { defaultRuntime } from "../../runtime.js";
import { readBackupRuns } from "../../state/backup-run-records.js";
import {
openOpenClawStateDatabase,
closeOpenClawStateDatabaseAsync,
} from "../../state/openclaw-state-db.js";
import { resolveOpenClawStateSqlitePath } from "../../state/openclaw-state-db.paths.js";
import { filesystemStorageProvider } from "../../storage/filesystem.js";
import { openStorageLocation } from "../../storage/locations.js";
import type { StorageBackend } from "../../storage/types.js";
import { withOpenClawTestState } from "../../test-utils/openclaw-test-state.js";
import { registerBackupCommand } from "./register.backup.js";
import { registerStorageCommand } from "./register.storage.js";
afterEach(() => vi.restoreAllMocks());
function createCliHarness() {
const writeJson = vi.spyOn(defaultRuntime, "writeJson").mockImplementation(() => {});
vi.spyOn(defaultRuntime, "log").mockImplementation(() => {});
const errors = vi.spyOn(defaultRuntime, "error").mockImplementation(() => {});
vi.spyOn(defaultRuntime, "exit").mockImplementation((code) => {
throw new Error(`CLI exit ${code}`);
});
const run = async (...argv: string[]) => {
writeJson.mockClear();
const program = new Command().exitOverride();
registerBackupCommand(program);
registerStorageCommand(program);
await program.parseAsync([...argv, "--json"], { from: "user" });
return writeJson.mock.calls.at(-1)?.[0];
};
return { run, errors };
}
describe("offsite backup CLI", () => {
async function withNamespaceRace(
exercise: (fixture: {
run: ReturnType<typeof createCliHarness>["run"];
errors: ReturnType<typeof createCliHarness>["errors"];
namespaceDir: string;
scratchRoot: string;
takeOver: () => Promise<void>;
useNewOwner: () => void;
intercept: (configure: (backend: StorageBackend) => void) => void;
}) => Promise<void>,
) {
await withOpenClawTestState({ layout: "state-only" }, async (state) => {
const destination = state.path("offsite");
const scratchRoot = state.path("scratch");
await fs.mkdir(destination);
await fs.mkdir(scratchRoot);
vi.spyOn(os, "tmpdir").mockReturnValue(scratchRoot);
await state.writeConfig({
storage: {
locations: {
archive: {
provider: "filesystem",
settings: { path: destination },
encryption: { passphrase: "synthetic-backup-test-passphrase" },
},
},
},
});
const { run, errors } = createCliHarness();
await run("storage", "init", "archive");
const identity = await deviceIdentity.loadOrCreateProcessDeviceIdentityAsync();
const nextIdentity = { ...identity, deviceId: "replacement-installation-device-id" };
const location = await openStorageLocation({ name: "archive", config: getRuntimeConfig() });
const scoped = location.scope("backups/test-host");
try {
await claimBackupNamespace(scoped, "test-host", identity.deviceId, false);
const open = filesystemStorageProvider.open;
await exercise({
run,
errors,
namespaceDir: path.join(destination, "backups", "test-host"),
scratchRoot,
takeOver: () => claimBackupNamespace(scoped, "test-host", nextIdentity.deviceId, true),
useNewOwner: () => {
vi.spyOn(deviceIdentity, "loadOrCreateProcessDeviceIdentityAsync").mockResolvedValue(
nextIdentity,
);
},
intercept: (configure) => {
vi.spyOn(filesystemStorageProvider, "open").mockImplementation(async (params) => {
const backend = await open(params);
configure(backend);
return backend;
});
},
});
} finally {
await location.close();
}
});
}
it("rejects publication after namespace takeover during streaming and lets the new owner create", async () => {
await withNamespaceRace(async (fixture) => {
const before = await fs.readdir(fixture.namespaceDir);
let displaced = false;
fixture.intercept((backend) => {
const put = backend.putObject;
backend.putObject = (key, body, opts) =>
put(
key,
(async function* () {
for await (const chunk of body) {
yield chunk;
if (key.endsWith(".tar.gz") && !displaced) {
displaced = true;
await fixture.takeOver();
}
}
})(),
opts,
);
});
const create = () =>
fixture.run(
"backup",
"create",
"--to",
"archive",
"--namespace",
"test-host",
"--only-config",
);
await expect(create()).rejects.toThrow("CLI exit 1");
expect(displaced).toBe(true);
expect(await fs.readdir(fixture.namespaceDir)).toEqual(before);
expect(await fs.readdir(fixture.scratchRoot)).toEqual([]);
expect((await readBackupRuns(process.env))[0]).toMatchObject({
status: "failed",
target: "archive",
namespace: "test-host",
error: expect.stringContaining("belongs to another OpenClaw installation"),
});
expect(fixture.errors.mock.calls.flat().join(" ")).toContain(
"belongs to another OpenClaw installation",
);
fixture.useNewOwner();
const created = (await create()) as OffsiteBackupResult;
expect(created.verified).toBe(true);
expect(await fs.readdir(fixture.namespaceDir)).toEqual(
expect.arrayContaining([...before, created.location!.key]),
);
expect((await readBackupRuns(process.env))[0]).toMatchObject({ status: "ok" });
expect(await fs.readdir(fixture.scratchRoot)).toEqual([]);
});
});
it("stops retention when namespace takeover follows the first delete's ownership check", async () => {
await withNamespaceRace(async (fixture) => {
const expired = [
"20260101T000000Z-11111111.tar.gz",
"20260102T000000Z-22222222.tar.gz",
"20260103T000000Z-33333333.tar.gz",
];
for (const key of expired) {
await fs.writeFile(path.join(fixture.namespaceDir, key), Buffer.alloc(200));
}
const deleted: string[] = [];
let nextMarker = false;
let displaced = false;
fixture.intercept((backend) => {
const get = backend.getObject;
const remove = backend.deleteObject;
backend.getObject = async (key, opts) => {
if (key === "openclaw-storage.json" && nextMarker) {
nextMarker = false;
displaced = true;
await fixture.takeOver();
}
const body = await get(key, opts);
if (key.endsWith("/owner.json") && deleted.length === 1 && !displaced) {
// Let the early claim read finish, then take over during the delete's marker check.
nextMarker = true;
}
return body;
};
backend.deleteObject = async (key, opts) => {
await remove(key, opts);
if (key.endsWith(".tar.gz")) {
deleted.push(path.basename(key));
}
};
});
await expect(
fixture.run(
"backup",
"create",
"--to",
"archive",
"--namespace",
"test-host",
"--only-config",
"--keep-daily",
"0",
),
).rejects.toThrow("CLI exit 1");
expect(displaced).toBe(true);
expect(deleted).toHaveLength(1);
expect(await fs.readdir(fixture.namespaceDir)).toEqual(
expect.arrayContaining(expired.filter((key) => !deleted.includes(key))),
);
expect((await readBackupRuns(process.env))[0]).toMatchObject({
status: "failed",
namespace: "test-host",
error: expect.stringContaining("belongs to another OpenClaw installation"),
});
expect(await fs.readdir(fixture.scratchRoot)).toEqual([]);
});
});
it("refuses uninitialized storage, then uploads, lists, verifies and stages an encrypted archive", async () => {
await withOpenClawTestState({ layout: "state-only" }, async (state) => {
const destination = state.path("offsite");
const scratchRoot = state.path("scratch");
await fs.mkdir(scratchRoot);
vi.spyOn(os, "tmpdir").mockReturnValue(scratchRoot);
vi.spyOn(os, "hostname").mockReturnValue("default-host");
await state.writeConfig({
agents: { entries: { main: { workspace: state.workspaceDir } } },
storage: {
locations: {
archive: {
provider: "filesystem",
settings: { path: destination },
encryption: { passphrase: "synthetic-backup-test-passphrase" },
},
},
},
});
await fs.writeFile(state.statePath("operator-note.txt"), "preserved state\n");
await fs.writeFile(
path.join(state.workspaceDir, "workspace-note.txt"),
"excluded workspace\n",
);
const { run, errors } = createCliHarness();
openOpenClawStateDatabase();
await closeOpenClawStateDatabaseAsync();
expect((await fs.stat(resolveOpenClawStateSqlitePath())).isFile()).toBe(true);
const localCopy = state.path("retained.tar.gz");
await expect(
run("backup", "create", "--to", "archive", "--output", localCopy),
).rejects.toThrow();
const unavailableMessage =
"Storage directory is unavailable. Reconnect the disk and check the configured path; storage init requires an existing directory.";
expect(errors.mock.calls.flat().join(" ")).toContain(unavailableMessage);
expect((await readBackupRuns(process.env))[0]).toMatchObject({
kind: "archive",
target: "archive",
namespace: "default-host",
status: "failed",
error: unavailableMessage,
});
errors.mockClear();
await fs.mkdir(destination);
await expect(
run("backup", "create", "--to", "archive", "--output", localCopy),
).rejects.toThrow();
expect(errors.mock.calls.flat().join(" ")).toContain("openclaw storage init archive");
await expect(fs.stat(localCopy)).rejects.toMatchObject({ code: "ENOENT" });
expect(await fs.readdir(scratchRoot)).toEqual([]);
expect(
(await readBackupRuns(process.env))[0],
errors.mock.calls.flat().join("\n"),
).toMatchObject({
kind: "archive",
target: "archive",
status: "failed",
error: expect.stringContaining("storage init archive"),
});
await run("storage", "init", "archive");
const namespaceDir = path.join(destination, "backups", "test-host");
const otherDir = path.join(destination, "backups", "test-host-other");
await fs.mkdir(namespaceDir, { recursive: true });
await fs.mkdir(otherDir, { recursive: true });
const oldKey = "20260101T000000Z-11111111.tar.gz";
await fs.writeFile(path.join(namespaceDir, oldKey), Buffer.alloc(200));
await fs.writeFile(path.join(namespaceDir, "foreign.txt"), "foreign");
await fs.writeFile(path.join(otherDir, oldKey), Buffer.alloc(200));
const location = await openStorageLocation({ name: "archive", config: getRuntimeConfig() });
const scoped = location.scope("backups/test-host");
const foreignClaim = {
version: 1,
deviceId: "another-installation-device-id",
hostname: "test-host",
claimedAt: 1,
};
const writeClaim = async (claim: typeof foreignClaim) => {
const bytes = Buffer.from(JSON.stringify(claim));
await scoped.putObject(
"owner.json",
(async function* () {
yield bytes;
})(),
{
sizeBytes: bytes.length,
},
);
};
await writeClaim(foreignClaim);
errors.mockClear();
await expect(
run(
"backup",
"create",
"--to",
"archive",
"--namespace",
"test-host",
"--output",
localCopy,
"--keep-daily",
"0",
),
).rejects.toThrow();
const collisionMessage =
'Backup namespace "test-host" in archive belongs to another OpenClaw installation (test-host, device another-inst). Pass --namespace <name> to use a separate namespace, or --claim-namespace to take it over deliberately (for example after moving to new hardware).';
expect(errors.mock.calls.flat().join(" ")).toContain(collisionMessage);
expect((await readBackupRuns(process.env))[0]).toMatchObject({
status: "failed",
target: "archive",
namespace: "test-host",
error: collisionMessage,
});
await expect(fs.stat(localCopy)).rejects.toMatchObject({ code: "ENOENT" });
expect(await fs.readdir(scratchRoot)).toEqual([]);
expect(await fs.readdir(namespaceDir)).toContain(oldKey);
const created = (await run(
"backup",
"create",
"--to",
"archive",
"--namespace",
"test-host",
"--claim-namespace",
"--no-include-workspace",
"--keep-daily",
"0",
)) as OffsiteBackupResult;
expect(created).toMatchObject({
verified: true,
localArchiveRetained: false,
retention: { kept: 1, deleted: 1 },
});
expect(created.location?.storedBytes).toBeGreaterThan(created.location!.plaintextBytes);
expect(await fs.readdir(scratchRoot)).toEqual([]);
expect(await fs.readdir(namespaceDir)).toEqual(
expect.arrayContaining(["foreign.txt", created.location!.key]),
);
await expect(fs.stat(path.join(namespaceDir, oldKey))).rejects.toMatchObject({
code: "ENOENT",
});
expect(await fs.readdir(otherDir)).toEqual([oldKey]);
const stored = await fs.readFile(path.join(namespaceDir, created.location!.key));
expect(stored.subarray(0, 8).toString()).toBe("OCSTOR1\n");
expect(
(await fs.readFile(path.join(namespaceDir, "owner.json"))).subarray(0, 8).toString(),
).toBe("OCSTOR1\n");
expect(await json((await scoped.getObject("owner.json"))!)).toMatchObject({
version: 1,
deviceId: (await loadDeviceIdentityIfPresentAsync())!.deviceId,
hostname: os.hostname(),
claimedAt: expect.any(Number),
});
// Recovery must work without this installation owning the namespace.
await scoped.delete("owner.json");
await writeClaim(foreignClaim);
await location.close();
await fs.writeFile(path.join(otherDir, "owner.json"), "x");
expect(await run("backup", "list", "--from", "archive")).toMatchObject({
namespaces: expect.arrayContaining([
{ namespace: "test-host", hostname: "test-host" },
{ namespace: "test-host-other" },
]),
});
const listed = await run("backup", "list", "--from", "archive", "--namespace", "test-host");
expect(listed).toMatchObject({
backups: [{ key: created.location!.key, sizeBytes: created.location!.plaintextBytes }],
});
const verified = await run(
"backup",
"verify",
"latest",
"--from",
"archive",
"--namespace",
"test-host",
);
expect(verified).toMatchObject({ ok: true });
const target = state.path("staged");
await run(
"backup",
"restore",
"latest",
"--from",
"archive",
"--namespace",
"test-host",
"--target",
target,
);
const capturedState = created.assets.find((asset) => asset.kind === "state");
expect(capturedState).toBeDefined();
const restoredState = path.join(target, capturedState!.archivePath);
expect(await fs.readFile(path.join(restoredState, "operator-note.txt"), "utf8")).toBe(
"preserved state\n",
);
expect(await fs.readFile(path.join(restoredState, "openclaw.json"), "utf8")).toBe(
await fs.readFile(state.configPath, "utf8"),
);
expect(created.assets.some((asset) => asset.kind === "workspace")).toBe(false);
await expect(
run(
"backup",
"restore",
"latest",
"--from",
"archive",
"--namespace",
"test-host",
"--target",
target,
),
).rejects.toThrow();
const configOnly = (await run(
"backup",
"create",
"--to",
"archive",
"--namespace",
"config",
"--only-config",
"--output",
localCopy,
)) as OffsiteBackupResult;
expect(configOnly).toMatchObject({
onlyConfig: true,
verified: true,
localArchiveRetained: true,
assets: [{ kind: "config" }],
});
expect((await fs.stat(localCopy)).size).toBe(configOnly.location?.plaintextBytes);
await fs.writeFile(path.join(destination, "backups", "config", "owner.json"), "x");
expect(
await run(
"backup",
"create",
"--to",
"archive",
"--namespace",
"config",
"--claim-namespace",
"--only-config",
),
).toMatchObject({ verified: true });
const reopened = await openStorageLocation({ name: "archive", config: getRuntimeConfig() });
try {
expect(
await json((await reopened.scope("backups/config").getObject("owner.json"))!),
).toMatchObject({
deviceId: (await loadDeviceIdentityIfPresentAsync())!.deviceId,
});
} finally {
await reopened.close();
}
});
});
});

83
src/cli/program/register.backup.ts
View file

@ -21,6 +21,15 @@ export function registerBackupCommand(program: Command) {
.command("create")
.description("Write a backup archive for config, credentials, sessions, and workspaces")
.option("--output <path>", "Archive path or destination directory")
.option("--to <location>", "Upload the verified archive to a storage location")
.option(
"--claim-namespace",
"Deliberately take over the backup namespace for this installation",
)
.option("--namespace <name>", "Backup namespace (default: sanitized hostname)")
.option("--keep-daily <n>", "Retain the newest backup in N daily UTC buckets")
.option("--keep-weekly <n>", "Retain the newest backup in N weekly UTC buckets")
.option("--keep-monthly <n>", "Retain the newest backup in N monthly UTC buckets")
.option("--json", "Output JSON", false)
.option("--dry-run", "Print the backup plan without writing the archive", false)
.option("--verify", "Verify the archive after writing it", false)
@ -60,6 +69,8 @@ export function registerBackupCommand(program: Command) {
backup
.command("verify <archive>")
.description("Validate a backup archive and its embedded manifest")
.option("--from <location>", "Read a backup key or latest from a storage location")
.option("--namespace <name>", "Backup namespace (default: sanitized hostname)")
.option("--json", "Output JSON", false)
.addHelpText(
"after",
@ -77,14 +88,24 @@ export function registerBackupCommand(program: Command) {
)
.action(async (archive, opts) => {
await runCommandWithRuntime(defaultRuntime, async () => {
const { backupVerifyCommand } = await import("../../commands/backup-verify.js");
await backupVerifyCommand(defaultRuntime, { ...opts, archive });
if (opts.from !== undefined) {
const { backupRemoteVerifyCommand } = await import("../../commands/backup-remote.js");
await backupRemoteVerifyCommand(defaultRuntime, { ...opts, archive });
} else {
if (opts.namespace) {
throw new Error("--namespace requires --from <location>.");
}
const { backupVerifyCommand } = await import("../../commands/backup-verify.js");
await backupVerifyCommand(defaultRuntime, { ...opts, archive });
}
});
});
backup
.command("restore <archive>")
.description("Restore a verified backup archive to a fresh staging directory")
.option("--from <location>", "Read a backup key or latest from a storage location")
.option("--namespace <name>", "Backup namespace (default: sanitized hostname)")
.requiredOption("--target <dir>", "Fresh target directory; non-empty directories are refused")
.option("--json", "Output JSON", false)
.addHelpText(
@ -103,8 +124,44 @@ export function registerBackupCommand(program: Command) {
)
.action(async (archive, opts) => {
await runCommandWithRuntime(defaultRuntime, async () => {
const { backupRestoreCommand } = await import("../../commands/backup-restore.js");
await backupRestoreCommand(defaultRuntime, { ...opts, archive });
if (opts.from !== undefined) {
const { backupRemoteRestoreCommand } = await import("../../commands/backup-remote.js");
await backupRemoteRestoreCommand(defaultRuntime, { ...opts, archive });
} else {
if (opts.namespace) {
throw new Error("--namespace requires --from <location>.");
}
const { backupRestoreCommand } = await import("../../commands/backup-restore.js");
await backupRestoreCommand(defaultRuntime, { ...opts, archive });
}
});
});
backup
.command("list")
.description("List archives in a storage location")
.requiredOption("--from <location>", "Storage location name")
.option("--namespace <name>", "Backup namespace (default: sanitized hostname)")
.option("--json", "Output JSON", false)
.action(async (opts) => {
await runCommandWithRuntime(defaultRuntime, async () => {
const { backupListCommand } = await import("../../commands/backup-remote.js");
await backupListCommand(defaultRuntime, opts);
});
});
backup
.command("record")
.description("Record an external backup job outcome")
.requiredOption("--status <status>", "ok or failed")
.requiredOption("--target <label>", "External backup target label")
.option("--bytes <n>", "Backup size in bytes")
.option("--error <text>", "Failure details")
.option("--json", "Output JSON", false)
.action(async (opts) => {
await runCommandWithRuntime(defaultRuntime, async () => {
const { backupRecordCommand } = await import("../../commands/backup-record.js");
await backupRecordCommand(defaultRuntime, opts);
});
});
@ -117,8 +174,18 @@ function registerBackupScheduleCommands(backup: Command): void {
addGatewayClientOptions(
backup
.command("enable")
.description("Provision a Gateway automation for scheduled Git backups")
.requiredOption("--repository <path>", "Git backup repository directory")
.description("Provision a Gateway automation for offsite or Git backups")
.option("--repository <path>", "Git backup repository directory")
.option("--to <location>", "Storage location for offsite archive backups")
.option(
"--claim-namespace",
"Deliberately take over the backup namespace on each scheduled run",
)
.option("--namespace <name>", "Backup namespace (default: sanitized hostname)")
.option("--no-include-workspace", "Exclude workspace directories from offsite archives")
.option("--keep-daily <n>", "Retain the newest backup in N daily UTC buckets")
.option("--keep-weekly <n>", "Retain the newest backup in N weekly UTC buckets")
.option("--keep-monthly <n>", "Retain the newest backup in N monthly UTC buckets")
.option("--every <duration>", "Backup interval", "24h")
.option("--push", "Push the current branch to origin after each backup", false)
.option("--exclude-secrets", "Omit credential-bearing database tables", false)
@ -140,7 +207,9 @@ function registerBackupScheduleCommands(backup: Command): void {
addGatewayClientOptions(
backup
.command("disable")
.description("Remove the scheduled Git backup automation")
.description("Remove both scheduled backup modes, or the selected mode")
.option("--offsite", "Disable only offsite archive backups", false)
.option("--git", "Disable only Git backups", false)
.action(async (opts) => {
await runCommandWithRuntime(defaultRuntime, async () => {
const { backupDisableCommand } = await import("../../commands/backup-schedule.js");

144
src/cli/program/register.storage.test.ts Normal file
View file

@ -0,0 +1,144 @@
import fs from "node:fs/promises";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import type { StorageProvider } from "../../storage/types.js";
import { OpenClawCommand } from "./openclaw-command.js";
import { registerStorageCommand } from "./register.storage.js";
const fixture = vi.hoisted(() => {
const config: OpenClawConfig = {};
return {
config,
runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() },
acquire: vi.fn(),
release: vi.fn(async () => {}),
};
});
vi.mock("../../config/config.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../../config/config.js")>()),
getRuntimeConfig: () => fixture.config,
}));
vi.mock("../../runtime.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../../runtime.js")>()),
defaultRuntime: fixture.runtime,
}));
vi.mock("../../storage/provider.js", async (importOriginal) => {
const original = await importOriginal<typeof import("../../storage/provider.js")>();
return {
...original,
acquireStorageProvider: (params: Parameters<typeof original.acquireStorageProvider>[0]) =>
params.providerId === "filesystem" || params.registry
? original.acquireStorageProvider(params)
: fixture.acquire(params),
};
});
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
let root: string;
beforeEach(() => {
vi.clearAllMocks();
root = tempDirs.make("openclaw-storage-cli-");
fixture.config = {
storage: {
locations: {
archive: { provider: "filesystem", settings: { path: root }, encryption: "none" },
},
},
};
});
async function runStorageCli(args: string[]) {
const program = new OpenClawCommand();
program.enablePositionalOptions();
registerStorageCommand(program);
await program.parseAsync(["storage", ...args], { from: "user" });
}
describe("storage CLI with filesystem transport", () => {
it("initializes, verifies a probe round trip, deletes the probe, and lists availability", async () => {
await runStorageCli(["--json", "init", "archive"]);
expect(fixture.runtime.error).not.toHaveBeenCalled();
expect(JSON.parse(String(fixture.runtime.log.mock.lastCall?.[0]))).toMatchObject({
name: "archive",
provider: "filesystem",
state: "ok",
encrypted: false,
});
await runStorageCli(["test", "archive", "--json"]);
expect(fixture.runtime.error).not.toHaveBeenCalled();
expect(JSON.parse(String(fixture.runtime.log.mock.lastCall?.[0]))).toMatchObject({
name: "archive",
state: "ok",
sizeBytes: 256,
});
expect(await fs.readdir(root)).toEqual(["openclaw-storage.json"]);
await runStorageCli(["list", "--json"]);
expect(JSON.parse(String(fixture.runtime.log.mock.lastCall?.[0]))).toMatchObject({
locations: [{ name: "archive", provider: "filesystem", state: "ok" }],
});
expect(fixture.runtime.exit).not.toHaveBeenCalled();
});
it.each([false, true])(
"lists plugin display targets even when opening fails (json=%s)",
async (json) => {
const provider: StorageProvider = {
id: "r2",
label: "R2",
describeTarget: (settings) => `r2://${String(settings.bucket)}/${String(settings.prefix)}`,
open: async () => {
throw new Error("synthetic-private-credential");
},
};
fixture.config = {
storage: {
locations: {
r2test: {
provider: "r2",
settings: { bucket: "bucket", prefix: "prefix" },
encryption: "none",
},
},
},
};
fixture.acquire.mockResolvedValue({
provider,
registry: {
storageProviders: new Map([["r2", { pluginId: "cloudflare", source: "test", provider }]]),
},
release: fixture.release,
});
await runStorageCli(["list", ...(json ? ["--json"] : [])]);
if (json) {
expect(JSON.parse(String(fixture.runtime.log.mock.lastCall?.[0]))).toMatchObject({
locations: [
{ name: "r2test", provider: "r2", displayTarget: "r2://bucket/prefix", state: "error" },
],
});
} else {
expect(fixture.runtime.log).toHaveBeenCalledWith(
expect.stringContaining("r2test (r2): error — r2://bucket/prefix"),
);
}
expect(fixture.runtime.log.mock.calls.flat().join(" ")).not.toContain(
"synthetic-private-credential",
);
expect(fixture.acquire).toHaveBeenCalledTimes(1);
expect(fixture.release).toHaveBeenCalledTimes(1);
},
);
it("refuses to test an uninitialized location without writing anything", async () => {
await runStorageCli(["test", "archive"]);
expect(fixture.runtime.exit).toHaveBeenCalledWith(1);
expect(fixture.runtime.error).toHaveBeenCalledWith(
expect.stringContaining("storage init archive"),
);
expect(await fs.readdir(root)).toEqual([]);
});
});

57
src/cli/program/register.storage.ts Normal file
View file

@ -0,0 +1,57 @@
import type { Command } from "commander";
import { formatDocsLink } from "../../../packages/terminal-core/src/links.js";
import { defaultRuntime } from "../../runtime.js";
import { runCommandWithRuntime } from "../cli-utils.js";
import { inheritOptionFromParent } from "../command-options.js";
import { applyParentDefaultHelpAction } from "./parent-default-help.js";
export function registerStorageCommand(program: Command): void {
const storage = program
.command("storage")
.description("List, initialize, and test configured storage locations")
.option("--json", "Output JSON", false)
.addHelpText(
"after",
`\nDocs: ${formatDocsLink("/cli/storage", "docs.openclaw.ai/cli/storage")}\n`,
);
applyParentDefaultHelpAction(storage);
storage
.command("list")
.description("List storage locations and probe their availability")
.option("--json", "Output JSON", false)
.action(async (opts: { json?: boolean }, command: Command) => {
await runCommandWithRuntime(defaultRuntime, async () => {
const { storageListCommand } = await import("../../commands/storage.js");
await storageListCommand(defaultRuntime, {
json: (inheritOptionFromParent<boolean>(command, "json") ?? opts.json) === true,
});
});
});
storage
.command("init <name>")
.description("Initialize a new location or verify its existing marker and encryption key")
.option("--json", "Output JSON", false)
.action(async (name: string, opts: { json?: boolean }, command: Command) => {
await runCommandWithRuntime(defaultRuntime, async () => {
const { storageInitCommand } = await import("../../commands/storage.js");
await storageInitCommand(defaultRuntime, name, {
json: (inheritOptionFromParent<boolean>(command, "json") ?? opts.json) === true,
});
});
});
storage
.command("test <name>")
.description("Write, read, verify, and delete a temporary probe object")
.option("--json", "Output JSON", false)
.action(async (name: string, opts: { json?: boolean }, command: Command) => {
await runCommandWithRuntime(defaultRuntime, async () => {
const { storageTestCommand } = await import("../../commands/storage.js");
await storageTestCommand(defaultRuntime, name, {
json: (inheritOptionFromParent<boolean>(command, "json") ?? opts.json) === true,
});
});
});
}

73
src/commands/backup-health.ts
View file

@ -1,6 +1,16 @@
import { note } from "../../packages/terminal-core/src/note.js";
import { formatCliCommand } from "../cli/command-format.js";
import { readBackupRunFreshness, type BackupRunFreshness } from "../state/backup-run-records.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { summarizeBackupSchedules, type BackupScheduleSummary } from "../cron/backup-command.js";
import { resolveCronJobsStorePathFromConfig } from "../cron/store/paths.js";
import { loadCronJobsStoreWithConfigJobsReadOnly } from "../cron/store/read-only.js";
import {
readBackupRuns,
summarizeBackupFreshness,
summarizeBackupTargets,
type BackupRunRecord,
type BackupRunFreshness,
} from "../state/backup-run-records.js";
// Backups older than two weeks no longer provide a useful routine recovery point.
const BACKUP_STALE_AFTER_MS = 14 * 24 * 60 * 60 * 1_000;
@ -47,10 +57,61 @@ function buildBackupDoctorHint(params: {
].join("\n");
}
/** Emit the non-repairing backup freshness hint when it applies. */
export async function noteBackupDoctorHint(env: NodeJS.ProcessEnv): Promise<void> {
const hint = buildBackupDoctorHint({ freshness: await readBackupRunFreshness(env) });
if (hint) {
note(hint, "Backups");
/** Report each scheduled destination independently of other successful backups. */
function buildOffsiteBackupDoctorHints(params: {
runs: readonly BackupRunRecord[];
schedules: readonly BackupScheduleSummary[];
now?: number;
}): string[] {
const targets = summarizeBackupTargets(params.runs);
const now = params.now ?? Date.now();
return params.schedules.flatMap((schedule) => {
if (schedule.mode !== "offsite" || !schedule.enabled) {
return [];
}
const target = targets.find(
(entry) =>
entry.kind === "archive" &&
entry.target === schedule.target &&
entry.namespace === schedule.namespace,
);
const failed = target?.latest.status === "failed";
const stale = !target?.latestOk || now - target.latestOk.createdAt > 3 * schedule.everyMs;
if (!failed && !stale) {
return [];
}
const reason = failed
? `The newest offsite backup attempt to ${schedule.target} failed${target.latest.error ? `: ${target.latest.error}` : "."}`
: target?.latestOk
? `The newest successful offsite backup to ${schedule.target} is older than three scheduled intervals.`
: `No successful offsite backup to ${schedule.target} is recorded.`;
return [
`${reason}\nCheck the destination with ${formatCliCommand(`openclaw storage test ${schedule.target}`)}.`,
];
});
}
/** Emit non-repairing freshness hints; configuration and ledger reads never probe destinations. */
export async function noteBackupDoctorHint(
env: NodeJS.ProcessEnv,
cfg?: OpenClawConfig,
): Promise<void> {
const runs = await readBackupRuns(env);
const hint = buildBackupDoctorHint({ freshness: summarizeBackupFreshness(runs) });
const hints = hint ? [hint] : [];
if (cfg) {
const loaded = await loadCronJobsStoreWithConfigJobsReadOnly(
resolveCronJobsStorePathFromConfig(cfg, env),
env,
);
hints.push(
...buildOffsiteBackupDoctorHints({
runs,
schedules: summarizeBackupSchedules(loaded.store.jobs),
}),
);
}
if (hints.length) {
note(hints.join("\n\n"), "Backups");
}
}

132
src/commands/backup-namespace.ts Normal file
View file

@ -0,0 +1,132 @@
import os from "node:os";
import { z } from "zod";
import { parseRemoteBackupTimestamp, resolveBackupNamespace } from "../infra/backup-retention.js";
import type { StorageLocation } from "../storage/locations.js";
const OWNER_KEY = "owner.json";
const ownerSchema = z.object({
version: z.literal(1),
deviceId: z.string().min(1),
hostname: z.string().min(1),
claimedAt: z.number().int().nonnegative(),
});
type NamespaceOwner = z.infer<typeof ownerSchema>;
async function readNamespaceOwner(location: StorageLocation): Promise<NamespaceOwner | undefined> {
const body = await location.getObject(OWNER_KEY);
if (!body) {
return undefined;
}
const chunks: Buffer[] = [];
let size = 0;
for await (const chunk of body) {
size += chunk.byteLength;
if (size > 4096) {
throw new Error("Backup namespace owner exceeds 4096 bytes.");
}
chunks.push(Buffer.from(chunk));
}
try {
return ownerSchema.parse(JSON.parse(Buffer.concat(chunks).toString("utf8")));
} catch {
throw new Error("Invalid backup namespace owner.json.");
}
}
function assertOwner(
location: StorageLocation,
namespace: string,
deviceId: string,
owner: NamespaceOwner | undefined,
): void {
const name = location.describe().name;
if (!owner) {
throw new Error(
`Backup namespace "${namespace}" in ${name} lost its ownership claim. Retry the backup.`,
);
}
if (owner.deviceId !== deviceId) {
throw new Error(
`Backup namespace "${namespace}" in ${name} belongs to another OpenClaw installation (${owner.hostname}, device ${owner.deviceId.slice(0, 12)}). Pass --namespace <name> to use a separate namespace, or --claim-namespace to take it over deliberately (for example after moving to new hardware).`,
);
}
}
export async function claimBackupNamespace(
location: StorageLocation,
namespace: string,
deviceId: string,
takeover: boolean,
): Promise<void> {
if (takeover) {
// Explicit takeover must also work when the old claim cannot be decoded.
await location.delete(OWNER_KEY);
}
let owner = await readNamespaceOwner(location);
if (!owner) {
const bytes = Buffer.from(
JSON.stringify({ version: 1, deviceId, hostname: os.hostname(), claimedAt: Date.now() }),
);
try {
await location.putObject(
OWNER_KEY,
(async function* () {
yield bytes;
})(),
{ sizeBytes: bytes.length },
);
} catch (error) {
// A concurrent claimant may have won the no-overwrite publication.
owner = await readNamespaceOwner(location);
if (!owner) {
throw error;
}
}
owner ??= await readNamespaceOwner(location);
}
assertOwner(location, namespace, deviceId, owner);
}
export async function assertBackupNamespaceOwner(
location: StorageLocation,
namespace: string,
deviceId: string,
): Promise<void> {
assertOwner(location, namespace, deviceId, await readNamespaceOwner(location));
}
/** Discovery is read-only and incomplete claim metadata must never prevent recovery. */
export async function listBackupNamespaces(
backups: StorageLocation,
): Promise<Array<{ namespace: string; hostname?: string }>> {
const namespaces = new Set<string>();
for await (const object of backups.list(undefined, {
acceptKey: (key) => {
const [namespace, file, extra] = key.split("/");
if (
!namespace ||
!file ||
extra !== undefined ||
(file !== OWNER_KEY && parseRemoteBackupTimestamp(file) === undefined)
) {
return false;
}
try {
resolveBackupNamespace(namespace);
} catch {
return false;
}
// Discover claims by key even if their encrypted contents or size are damaged.
namespaces.add(namespace);
return file !== OWNER_KEY;
},
})) {
namespaces.add(object.key.slice(0, object.key.indexOf("/")));
}
const result: Array<{ namespace: string; hostname?: string }> = [];
for (const namespace of [...namespaces].toSorted()) {
const owner = await readNamespaceOwner(backups.scope(namespace)).catch(() => undefined);
result.push({ namespace, ...(owner ? { hostname: owner.hostname } : {}) });
}
return result;
}

45
src/commands/backup-record.ts Normal file
View file

@ -0,0 +1,45 @@
import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js";
import { recordBackupOutcomeBestEffort } from "./backup-shared.js";
export type BackupRecordOptions = {
status?: string;
target?: string;
bytes?: string | number;
error?: string;
json?: boolean;
};
/** Host-owned jobs can contribute outcomes without changing their execution owner. */
export async function backupRecordCommand(
runtime: RuntimeEnv,
opts: BackupRecordOptions,
): Promise<void> {
if (opts.status !== "ok" && opts.status !== "failed") {
throw new Error("--status must be ok or failed.");
}
const target = opts.target?.trim();
if (!target) {
throw new Error("Missing required --target label.");
}
const bytes = opts.bytes === undefined ? undefined : Number(opts.bytes);
if (
bytes !== undefined &&
(!Number.isSafeInteger(bytes) || bytes < 0 || String(opts.bytes).trim() === "")
) {
throw new Error("--bytes must be a nonnegative integer.");
}
const outcome = {
kind: "external" as const,
archivePath: target,
target,
status: opts.status,
...(bytes !== undefined ? { bytes } : {}),
...(opts.error ? { error: opts.error } : {}),
};
await recordBackupOutcomeBestEffort(runtime, outcome);
if (opts.json) {
writeRuntimeJson(runtime, outcome);
} else {
runtime.log(`External backup: ${target} (${opts.status})`);
}
}

265
src/commands/backup-remote.ts Normal file
View file

@ -0,0 +1,265 @@
import { createReadStream, createWriteStream } from "node:fs";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { pipeline } from "node:stream/promises";
import { getRuntimeConfig } from "../config/config.js";
import type { BackupCreateOptions, BackupCreateResult } from "../infra/backup-create.js";
import {
createRemoteBackupKey,
normalizeBackupRetention,
parseRemoteBackupTimestamp,
resolveBackupNamespace,
selectBackupRetention,
type BackupRetentionOptions,
} from "../infra/backup-retention.js";
import {
createBackupScratchDirectory,
finishBackupScratch,
type BackupScratch,
} from "../infra/backup-scratch.js";
import { loadOrCreateProcessDeviceIdentityAsync } from "../infra/device-identity-async.js";
import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js";
import type { BackupRunLocation, BackupRunRetention } from "../state/backup-run-records.js";
import {
openStorageLocation,
type StorageLocation,
type StorageLocationObjectInfo,
} from "../storage/locations.js";
import {
assertBackupNamespaceOwner,
claimBackupNamespace,
listBackupNamespaces,
} from "./backup-namespace.js";
export type OffsiteBackupOptions = BackupCreateOptions &
BackupRetentionOptions & {
to: string;
namespace?: string;
claimNamespace?: boolean;
};
export type OffsiteBackupResult = BackupCreateResult & {
location?: BackupRunLocation;
retention?: BackupRunRetention;
localArchiveRetained?: boolean;
};
type RemoteBackupOptions = { from: string; namespace?: string; json?: boolean };
async function openBackupLocation(name: string, namespace?: string): Promise<StorageLocation> {
const location = await openStorageLocation({ name, config: getRuntimeConfig() });
try {
const probe = await location.probe();
if (probe.state !== "ok") {
throw new Error(
probe.message ??
`Storage location ${name}: ${probe.state}. Run \`openclaw storage test ${name}\`.`,
);
}
return location.scope(namespace === undefined ? "backups" : `backups/${namespace}`);
} catch (error) {
await location.close();
throw error;
}
}
async function listBackupObjects(location: StorageLocation): Promise<StorageLocationObjectInfo[]> {
const objects = [];
for await (const object of location.list(undefined, {
acceptKey: (key) => parseRemoteBackupTimestamp(key) !== undefined,
})) {
objects.push(object);
}
return objects.toSorted((a, b) => b.key.localeCompare(a.key));
}
export async function createOffsiteBackupArchive(
opts: OffsiteBackupOptions,
): Promise<OffsiteBackupResult> {
const namespace = resolveBackupNamespace(opts.namespace);
const retention = normalizeBackupRetention(opts);
// Open and probe before allocating scratch or capturing any source data.
const location = await openBackupLocation(opts.to, namespace);
let scratch: BackupScratch | undefined;
let output: OffsiteBackupResult | undefined;
try {
const { createBackupArchive } = await import("../infra/backup-create.js");
if (opts.dryRun) {
return await createBackupArchive(opts);
}
const { deviceId } = await loadOrCreateProcessDeviceIdentityAsync();
await claimBackupNamespace(location, namespace, deviceId, opts.claimNamespace === true);
if (!opts.output) {
scratch = await createBackupScratchDirectory(os.tmpdir());
}
const result = await createBackupArchive({
...opts,
output: scratch ? path.join(scratch.directory, "archive.tar.gz") : opts.output,
});
const { verifyBackupArchive } = await import("./backup-verify.js");
await verifyBackupArchive(result.archivePath);
result.verified = true;
const plaintextBytes = (await fs.stat(result.archivePath)).size;
const key = createRemoteBackupKey(Date.parse(result.createdAt));
await assertBackupNamespaceOwner(location, namespace, deviceId);
const uploaded = await location.putObject(key, createReadStream(result.archivePath), {
sizeBytes: plaintextBytes,
precondition: () => assertBackupNamespaceOwner(location, namespace, deviceId),
});
const stored = await location.stat(key);
if (
!stored ||
stored.sizeBytes !== plaintextBytes ||
stored.storedBytes !== uploaded.storedBytes
) {
throw new Error(
`Stored backup size could not be confirmed for ${opts.to}/${namespace}/${key}. Run \`openclaw storage test ${opts.to}\`.`,
);
}
const description = location.describe();
output = {
...result,
archivePath: opts.output
? result.archivePath
: `storage://${opts.to}/backups/${namespace}/${key}`,
localArchiveRetained: Boolean(opts.output),
location: {
name: description.name,
provider: description.provider,
locationId: description.locationId,
key,
namespace,
plaintextBytes,
storedBytes: stored.storedBytes,
},
};
if (Object.values(retention).some((value) => value !== undefined)) {
const selected = selectBackupRetention(
(await listBackupObjects(location)).map((object) => object.key),
retention,
);
for (const expired of selected.deleted) {
await assertBackupNamespaceOwner(location, namespace, deviceId);
await location.delete(expired, {
precondition: () => assertBackupNamespaceOwner(location, namespace, deviceId),
});
}
output.retention = { kept: selected.kept.length, deleted: selected.deleted.length };
}
return output;
} finally {
try {
if (scratch) {
const warning = await finishBackupScratch(scratch, opts.log);
if (warning && output) {
output.warnings = [...(output.warnings ?? []), warning];
}
}
} finally {
await location.close();
}
}
}
export async function backupListCommand(runtime: RuntimeEnv, opts: RemoteBackupOptions) {
const namespace = resolveBackupNamespace(opts.namespace);
const backupsLocation = await openBackupLocation(opts.from);
const location = backupsLocation.scope(namespace);
try {
const namespaces =
opts.namespace === undefined ? await listBackupNamespaces(backupsLocation) : undefined;
const backups = (await listBackupObjects(location)).map((object) => ({
key: object.key,
sizeBytes: object.sizeBytes,
storedBytes: object.storedBytes,
createdAt: new Date(parseRemoteBackupTimestamp(object.key)!).toISOString(),
}));
const result = {
location: location.describe(),
namespace,
backups,
...(namespaces ? { namespaces } : {}),
};
if (opts.json) {
writeRuntimeJson(runtime, result);
} else {
if (namespaces) {
runtime.log(
namespaces.length
? `Available namespaces in ${opts.from}:\n${namespaces.map((entry) => `${entry.namespace}${entry.hostname ? ` (${entry.hostname})` : ""}`).join("\n")}`
: `No backup namespaces in ${opts.from}.`,
);
}
runtime.log(
backups.length
? backups
.map(
(backup) =>
`${backup.key} ${backup.sizeBytes} bytes (${backup.storedBytes} stored)`,
)
.join("\n")
: `No backups in ${opts.from}/${namespace}.`,
);
}
return result;
} finally {
await location.close();
}
}
async function withDownloadedBackup<T>(
runtime: RuntimeEnv,
opts: RemoteBackupOptions & { archive: string },
consume: (archive: string) => Promise<T>,
): Promise<T> {
const namespace = resolveBackupNamespace(opts.namespace);
if (opts.archive !== "latest" && parseRemoteBackupTimestamp(opts.archive) === undefined) {
throw new Error(
"Expected a backup key from `openclaw backup list --from <location>` or latest.",
);
}
const location = await openBackupLocation(opts.from, namespace);
let scratch: BackupScratch | undefined;
try {
const key =
opts.archive === "latest" ? (await listBackupObjects(location))[0]?.key : opts.archive;
if (!key) {
throw new Error(`No backups in ${opts.from}/${namespace}.`);
}
const body = await location.getObject(key);
if (!body) {
throw new Error(`Backup ${key} was not found in ${opts.from}/${namespace}.`);
}
scratch = await createBackupScratchDirectory(os.tmpdir());
const archive = path.join(scratch.directory, "archive.tar.gz");
await pipeline(body, createWriteStream(archive, { flags: "wx", mode: 0o600 }));
return await consume(archive);
} finally {
try {
if (scratch) {
await finishBackupScratch(scratch, (message) => runtime.error(message));
}
} finally {
await location.close();
}
}
}
export async function backupRemoteVerifyCommand(
runtime: RuntimeEnv,
opts: RemoteBackupOptions & { archive: string },
) {
return await withDownloadedBackup(runtime, opts, async (archive) => {
const { backupVerifyCommand } = await import("./backup-verify.js");
return await backupVerifyCommand(runtime, { archive, json: opts.json });
});
}
export async function backupRemoteRestoreCommand(
runtime: RuntimeEnv,
opts: RemoteBackupOptions & { archive: string; target?: string },
) {
return await withDownloadedBackup(runtime, opts, async (archive) => {
const { backupRestoreCommand } = await import("./backup-restore.js");
return await backupRestoreCommand(runtime, { archive, target: opts.target, json: opts.json });
});
}

196
src/commands/backup-schedule.test.ts
View file

@ -7,10 +7,11 @@ import { Command } from "commander";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { registerCronCli } from "../cli/cron-cli.js";
import { registerBackupCommand } from "../cli/program/register.backup.js";
import { summarizeBackupSchedules } from "../cron/backup-command.js";
import { CronService } from "../cron/service.js";
import { createCronStoreHarness, createNoopLogger } from "../cron/service.test-harness.js";
import type { CronListPageOptions } from "../cron/service/list-page-types.js";
import type { CronJobCreate, CronJobPatch } from "../cron/types.js";
import type { CronJob, CronJobCreate, CronJobPatch } from "../cron/types.js";
import { defaultRuntime } from "../runtime.js";
import { createTestGatewayScheduler } from "../test-utils/gateway-scheduler-clock.js";
import { createTestRuntime } from "./test-runtime-config-helpers.js";
@ -70,6 +71,15 @@ describe("scheduled backups", () => {
gatewayRpc.isImplicitLocalTarget.mockReset().mockResolvedValue(true);
configMocks.getRuntimeConfig.mockReset().mockReturnValue({
agents: { list: [{ id: "main" }, { id: "ops-team" }] },
storage: {
locations: {
archive: {
provider: "filesystem",
settings: { path: "/mnt/backups" },
encryption: "none",
},
},
},
});
});
@ -139,6 +149,186 @@ describe("scheduled backups", () => {
expect(spec.payload.argv).not.toContain("--all");
});
it.each([false, true])(
"round-trips offsite CLI options with explicit claim=%s through the persisted schedule and status projection",
async (claim) => {
let persisted: CronJobCreate | undefined;
gatewayRpc.call.mockImplementation(
async (method: string, _options: unknown, params: CronJobCreate) => {
if (method !== "cron.add") {
throw new Error(`unexpected method ${method}`);
}
persisted = params;
return { created: true, job: { id: "offsite-job" } };
},
);
vi.spyOn(defaultRuntime, "log").mockImplementation(() => {});
await runCli([
"backup",
"enable",
"--to",
"archive",
"--every",
"6h",
"--namespace",
"host-a",
...(claim ? ["--claim-namespace"] : []),
"--no-include-workspace",
"--keep-daily",
"7",
"--keep-weekly",
"4",
"--keep-monthly",
"0",
]);
const spec = expectDefined(persisted, "persisted offsite schedule");
expect(spec.declarationKey).toBe("openclaw-backup-offsite-scheduled");
expect(spec.payload).toEqual({
kind: "command",
argv: [
"openclaw",
"backup",
"create",
"--to",
"archive",
"--namespace",
"host-a",
...(claim ? ["--claim-namespace"] : []),
"--no-include-workspace",
"--keep-daily",
"7",
"--keep-weekly",
"4",
"--keep-monthly",
"0",
],
});
expect(
summarizeBackupSchedules([
{
...spec,
id: "offsite-job",
createdAtMs: 1,
updatedAtMs: 1,
state: { nextRunAtMs: 21_600_001 },
},
]),
).toEqual([
{
id: "offsite-job",
mode: "offsite",
target: "archive",
namespace: "host-a",
enabled: true,
everyMs: 21_600_000,
nextRunAtMs: 21_600_001,
},
]);
},
);
it.each([{ argv: ["--all"] }, { argv: ["--global"] }, { argv: ["--agent", "main"] }])(
"reports installed Git schedule argv $argv in status",
({ argv }) => {
expect(
summarizeBackupSchedules([
{
id: "git-job",
name: "Renamed by operator",
enabled: true,
createdAtMs: 1,
updatedAtMs: 1,
sessionTarget: "isolated",
wakeMode: "now",
state: {},
declarationKey: "openclaw-backup-scheduled",
schedule: { kind: "every", everyMs: 86_400_000 },
payload: {
kind: "command",
argv: [
"openclaw",
"backup",
"git",
"create",
"--repository",
"/backups/git",
...argv,
"--push",
"--exclude-secrets",
],
},
},
]),
).toEqual([
{
id: "git-job",
mode: "git",
everyMs: 86_400_000,
target: "/backups/git",
enabled: true,
},
]);
},
);
it.each([
{ flags: [], removed: ["git-job", "offsite-job"] },
{ flags: ["--git"], removed: ["git-job"] },
{ flags: ["--offsite"], removed: ["offsite-job"] },
])("disables only selected backup modes $flags", async ({ flags, removed }) => {
const base: CronJob = {
id: "git-job",
name: "Renamed by operator",
enabled: true,
createdAtMs: 1,
updatedAtMs: 1,
declarationKey: "openclaw-backup-scheduled",
schedule: { kind: "every", everyMs: 60_000 },
payload: { kind: "command", argv: ["openclaw", "backup", "git", "create"] },
sessionTarget: "isolated",
wakeMode: "now",
state: {},
};
const jobs = [
base,
{ ...base, id: "offsite-job", declarationKey: "openclaw-backup-offsite-scheduled" },
{ ...base, id: "unmanaged-job", declarationKey: undefined },
];
gatewayRpc.call.mockImplementation(async (method: string) => {
if (method === "cron.list") {
return {
jobs,
snapshotRevision: "1",
total: jobs.length,
offset: 0,
limit: 200,
hasMore: false,
nextOffset: null,
};
}
if (method === "cron.remove") {
return { removed: true };
}
throw new Error(`unexpected method ${method}`);
});
vi.spyOn(defaultRuntime, "log").mockImplementation(() => {});
await runCli(["backup", "disable", ...flags]);
expect(
gatewayRpc.call.mock.calls
.filter(([method]) => method === "cron.remove")
.map((call) => call[2]),
).toEqual(removed.map((id) => ({ id })));
});
it.each([
{ options: { to: "archive", repository: "/backups" }, message: "cannot be combined" },
{ options: { repository: "/backups", keepDaily: "7" }, message: "require --to" },
{ options: { to: "missing" }, message: 'Storage location "missing" is not configured' },
])("rejects invalid schedule options $options", async ({ options, message }) => {
await expect(backupEnableCommand(createTestRuntime(), options)).rejects.toThrow(message);
expect(gatewayRpc.call).not.toHaveBeenCalled();
});
it.each([
[
"unknown",
@ -267,12 +457,12 @@ describe("scheduled backups", () => {
await runCli(["backup", "disable"]);
expect(await cron.readJob(managed.id)).toBeUndefined();
expect(runtime.log).toHaveBeenLastCalledWith("Scheduled Git backups disabled.");
expect(runtime.log).toHaveBeenLastCalledWith("Scheduled backups disabled.");
expect(
(await cron.list({ includeDisabled: true })).map((job) => job.id).toSorted(),
).toEqual(decoyIds.toSorted());
await runCli(["backup", "disable"]);
expect(runtime.log).toHaveBeenLastCalledWith("Scheduled Git backups are already disabled.");
expect(runtime.log).toHaveBeenLastCalledWith("Scheduled backups are already disabled.");
expect(runtime.error).not.toHaveBeenCalled();
expect(runJob).not.toHaveBeenCalled();
} finally {

161
src/commands/backup-schedule.ts
View file

@ -8,9 +8,15 @@ import {
import { parseDurationMs } from "../cli/parse-duration.js";
import { getRuntimeConfig } from "../config/config.js";
import {
SCHEDULED_BACKUP_COMMAND,
SCHEDULED_BACKUP_DECLARATION_KEY,
backupScheduleModeForDeclaration,
buildBackupScheduleJob,
type BackupScheduleSpec,
} from "../cron/backup-command.js";
import {
normalizeBackupRetention,
resolveBackupNamespace,
type BackupRetentionOptions,
} from "../infra/backup-retention.js";
import { executeGitCommand } from "../infra/git-exec.js";
import { normalizeAgentId } from "../routing/session-key.js";
import type { RuntimeEnv } from "../runtime.js";
@ -21,15 +27,22 @@ import { resolveRequiredBackupPath } from "./backup-shared.js";
const LOCAL_GATEWAY_REQUIRED_ERROR =
"backup enable manages backups on the Gateway host and currently requires a local Gateway. Create the cron job manually with openclaw cron add for remote Gateways.";
type BackupScheduleOptions = GatewayRpcOpts & {
repository?: string;
every?: string;
push?: boolean;
excludeSecrets?: boolean;
includeSecrets?: boolean;
globalOnly?: boolean;
agent?: string;
};
export type BackupScheduleOptions = GatewayRpcOpts &
BackupRetentionOptions & {
repository?: string;
every?: string;
push?: boolean;
excludeSecrets?: boolean;
includeSecrets?: boolean;
globalOnly?: boolean;
agent?: string;
to?: string;
namespace?: string;
claimNamespace?: boolean;
includeWorkspace?: boolean;
};
export type BackupDisableOptions = GatewayRpcOpts & { git?: boolean; offsite?: boolean };
/**
* Unattended pushed schedules make credential retention durable in remote
@ -47,11 +60,52 @@ function resolveScheduledRedaction(options: BackupScheduleOptions): boolean {
return options.includeSecrets !== true;
}
function buildScheduledArgv(
options: BackupScheduleOptions,
repositoryPath: string,
redactSecrets: boolean,
): string[] {
function resolveScheduleSpec(options: BackupScheduleOptions, everyMs: number): BackupScheduleSpec {
if (options.to !== undefined) {
if (
options.repository !== undefined ||
options.push ||
options.excludeSecrets ||
options.includeSecrets ||
options.globalOnly ||
options.agent !== undefined
) {
throw new Error(
"--to cannot be combined with Git backup options (--repository, --push, --exclude-secrets, --include-secrets, --global-only, --agent).",
);
}
const location = options.to.trim();
if (!location) {
throw new Error("--to must name a configured storage location.");
}
if (!getRuntimeConfig({ skipPluginValidation: true }).storage?.locations?.[location]) {
throw new Error(
`Storage location "${location}" is not configured. Run openclaw storage list.`,
);
}
return {
mode: "offsite",
everyMs,
location,
namespace: resolveBackupNamespace(options.namespace),
claimNamespace: options.claimNamespace === true,
includeWorkspace: options.includeWorkspace !== false,
...normalizeBackupRetention(options),
};
}
if (
options.namespace !== undefined ||
options.claimNamespace ||
options.includeWorkspace === false ||
options.keepDaily !== undefined ||
options.keepWeekly !== undefined ||
options.keepMonthly !== undefined
) {
throw new Error(
"--namespace, --claim-namespace, --no-include-workspace, and --keep-* require --to <location>.",
);
}
const repository = resolveRequiredBackupPath(options.repository, "--repository");
const agent = options.agent?.trim();
if (options.agent !== undefined && !agent) {
throw new Error("--agent must not be blank");
@ -65,14 +119,18 @@ function buildScheduledArgv(
normalizeAgentId(agent),
)
: undefined;
return [
...SCHEDULED_BACKUP_COMMAND,
"--repository",
repositoryPath,
...(options.globalOnly ? ["--global"] : agentId ? ["--agent", agentId] : ["--all"]),
...(options.push ? ["--push"] : []),
...(redactSecrets ? ["--exclude-secrets"] : []),
];
return {
mode: "git",
everyMs,
repository,
scope: options.globalOnly
? { kind: "global" }
: agentId
? { kind: "agent", agentId }
: { kind: "all" },
push: options.push === true,
excludeSecrets: resolveScheduledRedaction(options),
};
}
async function assertLocalGatewayScheduleTarget(options: GatewayRpcOpts): Promise<void> {
@ -88,41 +146,27 @@ export async function backupEnableCommand(
options: BackupScheduleOptions,
): Promise<{ id: string; updated: boolean }> {
await assertLocalGatewayScheduleTarget(options);
const repositoryPath = resolveRequiredBackupPath(options.repository, "--repository");
// Explicit blanks must reach duration validation instead of creating a default schedule.
const every = options.every?.trim() ?? "24h";
const everyMs = parseDurationMs(every, { defaultUnit: "ms" });
if (!Number.isSafeInteger(everyMs) || everyMs <= 0) {
throw new Error("--every must be a positive duration such as 6h or 24h.");
}
const redactSecrets = resolveScheduledRedaction(options);
const spec = {
declarationKey: SCHEDULED_BACKUP_DECLARATION_KEY,
name: SCHEDULED_BACKUP_DECLARATION_KEY,
enabled: true,
schedule: { kind: "every" as const, everyMs },
sessionTarget: "isolated" as const,
wakeMode: "now" as const,
payload: {
kind: "command" as const,
argv: buildScheduledArgv(options, repositoryPath, redactSecrets),
},
delivery: { mode: "none" as const },
};
if (options.push) {
const spec = resolveScheduleSpec(options, everyMs);
if (spec.mode === "git" && spec.push) {
// The unattended job cannot configure a remote; without this preflight the
// first scheduled run records a degraded push-failed backup instead.
const origin = await executeGitCommand(repositoryPath, ["remote", "get-url", "origin"]);
const origin = await executeGitCommand(spec.repository, ["remote", "get-url", "origin"]);
if (origin.code !== 0) {
throw new Error(
`--push requires an origin remote. Run: openclaw backup git init --repository ${shortenHomePath(repositoryPath)} --remote <url>`,
`--push requires an origin remote. Run: openclaw backup git init --repository ${shortenHomePath(spec.repository)} --remote <url>`,
);
}
if (!redactSecrets) {
if (!spec.excludeSecrets) {
runtime.error(GIT_BACKUP_PUSH_CREDENTIAL_WARNING);
}
}
const result = (await callGatewayFromCli("cron.add", options, spec)) as {
const result = (await callGatewayFromCli("cron.add", options, buildBackupScheduleJob(spec))) as {
created?: boolean;
updated?: boolean;
job?: { id?: string };
@ -133,23 +177,38 @@ export async function backupEnableCommand(
}
const updated = result.created === false;
runtime.log(
`Scheduled Git backups ${updated ? "updated" : "enabled"}: every ${every} to ${shortenHomePath(repositoryPath)}`,
`Scheduled ${spec.mode === "git" ? "Git" : "offsite"} backups ${updated ? "updated" : "enabled"}: every ${every} to ${spec.mode === "git" ? shortenHomePath(spec.repository) : spec.location}`,
);
return { id, updated };
}
export async function backupDisableCommand(
runtime: RuntimeEnv,
options: GatewayRpcOpts,
options: BackupDisableOptions,
): Promise<{ removed: boolean }> {
await assertLocalGatewayScheduleTarget(options);
if (options.git && options.offsite) {
throw new Error("Use either --git or --offsite, or omit both to disable all backup schedules.");
}
const { jobs } = await listCronJobsFromGateway(options, { includeDisabled: true });
const existing = jobs.find((job) => job.declarationKey === SCHEDULED_BACKUP_DECLARATION_KEY);
if (!existing) {
runtime.log("Scheduled Git backups are already disabled.");
const selectedMode = options.git ? "git" : options.offsite ? "offsite" : undefined;
const existing = jobs.filter((job) => {
const mode = backupScheduleModeForDeclaration(job.declarationKey);
return mode !== undefined && (selectedMode === undefined || mode === selectedMode);
});
const label =
selectedMode === "git"
? "Scheduled Git backups"
: selectedMode === "offsite"
? "Scheduled offsite backups"
: "Scheduled backups";
if (existing.length === 0) {
runtime.log(`${label} are already disabled.`);
return { removed: false };
}
await callGatewayFromCli("cron.remove", options, { id: existing.id });
runtime.log("Scheduled Git backups disabled.");
for (const job of existing) {
await callGatewayFromCli("cron.remove", options, { id: job.id });
}
runtime.log(`${label} disabled.`);
return { removed: true };
}

69
src/commands/backup.ts
View file

@ -1,35 +1,58 @@
import {
createBackupArchive,
type BackupCreateOptions,
type BackupCreateResult,
} from "../infra/backup-create.js";
import { createBackupArchive, type BackupCreateOptions } from "../infra/backup-create.js";
import { resolveBackupNamespace, type BackupRetentionOptions } from "../infra/backup-retention.js";
import { formatErrorMessage } from "../infra/errors.js";
import { beginLifecycleWriteCustody } from "../infra/lifecycle-write-custody.js";
import { withCommandProcessScope } from "../process/exec-spawn.js";
import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js";
import { createLazyPromise } from "../shared/lazy-promise.js";
import type { OffsiteBackupResult } from "./backup-remote.js";
import { recordBackupOutcomeBestEffort } from "./backup-shared.js";
import { formatBackupCreateSummary } from "./backup-summary.js";
const loadBackupVerifyRuntime = createLazyPromise(() => import("./backup-verify.js"));
const loadBackupRemoteRuntime = createLazyPromise(() => import("./backup-remote.js"));
export type BackupCommandCreateOptions = BackupCreateOptions &
BackupRetentionOptions & {
to?: string;
namespace?: string;
claimNamespace?: boolean;
};
/** Create a backup archive, optionally verify it, and emit text or JSON output. */
export async function backupCreateCommand(
runtime: RuntimeEnv,
opts: BackupCreateOptions = {},
): Promise<BackupCreateResult> {
let archivePath = opts.output ?? process.cwd();
opts: BackupCommandCreateOptions = {},
): Promise<OffsiteBackupResult> {
if (
opts.to === undefined &&
[opts.namespace, opts.claimNamespace, opts.keepDaily, opts.keepWeekly, opts.keepMonthly].some(
(value) => value !== undefined,
)
) {
throw new Error("--namespace, --claim-namespace, and retention flags require --to <location>.");
}
let archivePath = opts.output ?? (opts.to === undefined ? process.cwd() : `storage://${opts.to}`);
const releaseCustody = opts.dryRun ? undefined : beginLifecycleWriteCustody("backup");
let failure: unknown;
let namespace = opts.namespace;
try {
const result = await withCommandProcessScope(() =>
createBackupArchive({
if (opts.to !== undefined) {
namespace = resolveBackupNamespace(opts.namespace);
}
const result: OffsiteBackupResult = await withCommandProcessScope(async () => {
const options = {
...opts,
log: opts.log ?? (opts.json ? undefined : (message: string) => runtime.log(message)),
}),
);
};
if (opts.to !== undefined) {
const { createOffsiteBackupArchive } = await loadBackupRemoteRuntime();
return await createOffsiteBackupArchive({ ...options, to: opts.to, namespace });
}
return await createBackupArchive(options);
});
archivePath = result.archivePath;
if (opts.verify && !opts.dryRun) {
if (opts.verify && !opts.dryRun && !result.verified) {
const { verifyBackupArchive } = await loadBackupVerifyRuntime();
await verifyBackupArchive(result.archivePath);
result.verified = true;
@ -39,12 +62,31 @@ export async function backupCreateCommand(
kind: "archive",
archivePath,
status: "ok",
...(opts.to !== undefined
? {
target: opts.to,
namespace,
location: result.location,
retention: result.retention,
bytes: result.location?.plaintextBytes,
}
: {}),
});
}
if (opts.json) {
writeRuntimeJson(runtime, result);
} else {
runtime.log(formatBackupCreateSummary(result).join("\n"));
if (result.location) {
runtime.log(
`Uploaded to ${result.location.name}/backups/${result.location.namespace}/${result.location.key} (${result.location.storedBytes} stored bytes).${result.localArchiveRetained ? " Local archive retained." : ""}`,
);
}
if (result.retention) {
runtime.log(
`Retention: ${result.retention.kept} kept, ${result.retention.deleted} deleted.`,
);
}
}
return result;
} catch (error) {
@ -55,6 +97,7 @@ export async function backupCreateCommand(
archivePath,
status: "failed",
error: formatErrorMessage(error),
...(opts.to !== undefined ? { target: opts.to, namespace } : {}),
});
}
throw error;

4
src/commands/doctor-backup-scratch.test.ts
View file

@ -7,7 +7,7 @@ import { createBackupScratchDirectory, finishBackupScratch } from "../infra/back
import * as fsSafe from "../infra/fs-safe.js";
import { noteBackupScratchHealth } from "./doctor-backup-scratch.js";
const mocks = vi.hoisted(() => ({ note: vi.fn(), directories: vi.fn<() => string[]>() }));
const mocks = vi.hoisted(() => ({ note: vi.fn(), directories: vi.fn<() => Promise<string[]>>() }));
vi.mock("../../packages/terminal-core/src/note.js", () => ({ note: mocks.note }));
vi.mock("../state/backup-run-records.js", () => ({
readBackupArchiveDirectories: mocks.directories,
@ -20,7 +20,7 @@ it("reports without mutation and fixes abandoned scratch at recorded archive loc
const fallback = path.join(root, "archive-parent");
await fs.mkdir(fallback);
vi.spyOn(os, "tmpdir").mockReturnValue(root);
mocks.directories.mockReturnValue([fallback]);
mocks.directories.mockResolvedValue([fallback]);
const stale = await createBackupScratchDirectory(fallback);
const live = await createBackupScratchDirectory(root);
stale.release();

2
src/commands/doctor-backup-scratch.ts
View file

@ -10,7 +10,7 @@ export async function noteBackupScratchHealth(
): Promise<void> {
const roots = [os.tmpdir()];
try {
roots.push(...readBackupArchiveDirectories(env));
roots.push(...(await readBackupArchiveDirectories(env)));
} catch (error) {
note(
`Cannot discover recorded backup scratch locations: ${formatErrorMessage(error)}`,

26
src/commands/status-overview-rows.test.ts
View file

@ -33,6 +33,32 @@ function findRowValue(rows: Array<{ Item: string; Value: string }>, item: string
}
describe("status-overview-rows", () => {
it("shows the latest offsite attempt beside a newer local backup", () => {
vi.spyOn(Date, "now").mockReturnValue(3_600_000);
const rows = buildStatusCommandOverviewRows({
...createStatusCommandOverviewRowsParams(),
backupFreshness: {
latest: {
id: "local",
createdAt: 3_000_000,
archivePath: "/backup/local",
kind: "git",
status: "ok",
},
latestOffsite: {
id: "remote",
createdAt: 1,
archivePath: "",
kind: "archive",
status: "failed",
target: "offsite",
},
},
});
expect(findRowValue(rows, "Backups")).toContain("last ok");
expect(findRowValue(rows, "Offsite backup")).toContain("offsite: last attempt failed");
});
it.each(["default", "all"])("preserves service inspection failures in %s output", (mode) => {
const params = createStatusCommandOverviewRowsParams();
const service = {

13
src/commands/status-overview-rows.ts
View file

@ -188,6 +188,19 @@ export function buildStatusCommandOverviewRows(params: {
formatTimeAgo,
}),
},
...(params.backupFreshness.latestOffsite
? [
{
Item: "Offsite backup",
Value: `${params.backupFreshness.latestOffsite.location?.name ?? params.backupFreshness.latestOffsite.target}: ${buildBackupStatusValue(
{
freshness: { latest: params.backupFreshness.latestOffsite },
formatTimeAgo,
},
)}`,
},
]
: []),
{ Item: "Heartbeat", Value: heartbeatValue },
...(lastHeartbeatValue ? [{ Item: "Last heartbeat", Value: lastHeartbeatValue }] : []),
{

126
src/commands/storage.ts Normal file
View file

@ -0,0 +1,126 @@
import { randomBytes, randomUUID } from "node:crypto";
import { getRuntimeConfig } from "../config/config.js";
import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js";
import {
initStorageLocation,
listStorageLocations,
openStorageLocation,
probeStorageLocation,
storageLocationError,
type StorageProbeResult,
} from "../storage/locations.js";
import { acquireStorageProvider } from "../storage/provider.js";
type StorageCommandOptions = { json?: boolean };
export async function storageListCommand(runtime: RuntimeEnv, opts: StorageCommandOptions) {
const config = getRuntimeConfig();
const locations = [];
for (const location of listStorageLocations(config)) {
let displayTarget = location.displayTarget;
let probe: StorageProbeResult;
try {
const acquired = await acquireStorageProvider({ providerId: location.provider, config });
try {
const settings = config.storage?.locations?.[location.name]?.settings;
if (settings) {
displayTarget = acquired.provider.describeTarget?.(settings);
}
probe = await probeStorageLocation({
name: location.name,
config,
registry: acquired.registry,
});
} finally {
await acquired.release();
}
} catch (error) {
const { state, message } = storageLocationError(error);
probe = { state, message };
}
locations.push({
...location,
...(displayTarget === undefined ? {} : { displayTarget }),
...probe,
});
}
if (opts.json) {
writeRuntimeJson(runtime, { locations });
return;
}
if (locations.length === 0) {
runtime.log("No storage locations configured. Configure storage.locations to add one.");
}
for (const location of locations) {
runtime.log(
`${location.name} (${location.provider}): ${location.state}${location.displayTarget ? ` — ${location.displayTarget}` : ""}${location.message ? `\n ${location.message}` : ""}`,
);
}
}
export async function storageInitCommand(
runtime: RuntimeEnv,
name: string,
opts: StorageCommandOptions,
) {
const location = await initStorageLocation({ name, config: getRuntimeConfig() });
try {
const description = location.describe();
if (opts.json) {
writeRuntimeJson(runtime, { ...description, state: "ok" });
} else {
runtime.log(`Storage location ${name} initialized: ${description.displayTarget}`);
}
} finally {
await location.close();
}
}
export async function storageTestCommand(
runtime: RuntimeEnv,
name: string,
opts: StorageCommandOptions,
) {
const location = await openStorageLocation({ name, config: getRuntimeConfig() });
try {
const key = `.openclaw-probe-${randomUUID()}`;
const payload = randomBytes(256);
await location.putObject(
key,
(async function* () {
yield payload;
})(),
{ sizeBytes: payload.length },
);
try {
const body = await location.getObject(key);
if (!body) {
throw new Error(`Storage test for "${name}" failed: the written object is missing.`);
}
let offset = 0;
for await (const chunk of body) {
if (
offset + chunk.length > payload.length ||
!payload.subarray(offset, offset + chunk.length).equals(chunk)
) {
throw new Error(`Storage test for "${name}" failed: read-back bytes differ.`);
}
offset += chunk.length;
}
if (offset !== payload.length) {
throw new Error(`Storage test for "${name}" failed: read-back bytes are truncated.`);
}
} finally {
await location.delete(key);
}
if (opts.json) {
writeRuntimeJson(runtime, { ...location.describe(), state: "ok", sizeBytes: payload.length });
} else {
runtime.log(
`Storage location ${name}: wrote, read, verified, and deleted ${payload.length} bytes.`,
);
}
} finally {
await location.close();
}
}

2
src/config/plugin-auto-enable.materialize.ts
View file

@ -33,6 +33,8 @@ function resolvePluginAutoEnableCandidateReason(candidate: PluginAutoEnableCandi
return `${candidate.providerId} speech provider selected`;
case "worker-provider-selected":
return `${candidate.providerId} worker provider selected`;
case "storage-provider-selected":
return `${candidate.providerId} storage provider selected`;
case "decision-provider-selected":
return `${candidate.providerId} decision provider selected`;
case "agent-harness-runtime-configured":

59
src/config/plugin-auto-enable.providers.test.ts
View file

@ -136,6 +136,65 @@ describe("applyPluginAutoEnable providers", () => {
});
});
it("auto-enables the bundled owner selected by a storage location", () => {
const result = applyPluginAutoEnable({
config: {
storage: {
locations: {
archive: { provider: " ARCHIVE-OBJECTS ", settings: {}, encryption: "none" },
},
},
plugins: { allow: ["telegram"] },
},
env,
manifestRegistry: makeRegistry([
{
id: "storage-fixture",
channels: [],
contracts: { storageProviders: ["archive-objects"] },
origin: "bundled",
},
]),
});
expect(result.config.plugins?.entries?.["storage-fixture"]?.enabled).toBe(true);
expect(result.config.plugins?.allow).toEqual(["telegram", "storage-fixture"]);
expect(result.autoEnabledReasons).toEqual({
"storage-fixture": ["archive-objects storage provider selected"],
});
});
it.each([
{ origin: "global" as const, plugins: {} },
{ origin: "bundled" as const, plugins: { enabled: false } },
{ origin: "bundled" as const, plugins: { entries: { "storage-fixture": { enabled: false } } } },
{ origin: "bundled" as const, plugins: { deny: ["storage-fixture"] } },
])(
"does not auto-enable storage against external trust or explicit disablement: %j",
({ origin, plugins }) => {
const result = applyPluginAutoEnable({
config: {
storage: {
locations: {
archive: { provider: "archive-objects", settings: {}, encryption: "none" },
},
},
plugins,
},
env,
manifestRegistry: makeRegistry([
{
id: "storage-fixture",
channels: [],
contracts: { storageProviders: ["archive-objects"] },
origin,
},
]),
});
expect(result.config.plugins?.entries?.["storage-fixture"]?.enabled).not.toBe(true);
expect(result.changes).toEqual([]);
},
);
it("requires explicit enablement for external worker providers", () => {
const result = applyPluginAutoEnable({
config: { cloudWorkers: { profiles: { production: { provider: "cloud-vendor" } } } },

11
src/config/plugin-auto-enable.shared.ts
View file

@ -16,6 +16,10 @@ import { isNativeSessionCatalogOptOutOnly } from "../plugins/native-session-cata
import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
import { resolveOwningPluginIdsForModelRef } from "../plugins/providers.js";
import { resolvePluginSetupAutoEnableReasons } from "../plugins/setup-registry.js";
import {
collectConfiguredStorageProviderIds,
listBundledStorageProviderOwners,
} from "../plugins/storage-provider-manifest.js";
import { collectConfiguredWorkerProviderIds } from "../plugins/worker-provider-config.js";
import { listBundledWorkerProviderOwners } from "../plugins/worker-provider-manifest.js";
import { isKernelOwnedChannelConfigKey } from "./channel-config-keys.js";
@ -290,6 +294,7 @@ function hasConfiguredPluginProviders(cfg: OpenClawConfig): boolean {
hasConfiguredProviderModelOrHarness(cfg) ||
hasConfiguredVoiceProviderSelection(cfg) ||
collectConfiguredWorkerProviderIds(cfg).length > 0 ||
collectConfiguredStorageProviderIds(cfg).length > 0 ||
hasConfiguredWebSearchProviderSelection(cfg)
);
}
@ -411,6 +416,12 @@ export function resolveConfiguredPluginAutoEnableCandidates(
)) {
changes.push({ pluginId, kind: "worker-provider-selected", providerId });
}
for (const { pluginId, providerId } of listBundledStorageProviderOwners(
params.registry,
collectConfiguredStorageProviderIds(params.config),
)) {
changes.push({ pluginId, kind: "storage-provider-selected", providerId });
}
const decisionProviderIds = new Set(getConfiguredDecisionProviderIds(params.config));
for (const plugin of params.registry.plugins) {

1
src/config/plugin-auto-enable.test-helpers.ts
View file

@ -60,6 +60,7 @@ export function makeRegistry(
contracts?: {
speechProviders?: string[];
workerProviders?: string[];
storageProviders?: string[];
decisionProviders?: string[];
webSearchProviders?: string[];
webFetchProviders?: string[];

4
src/config/plugin-auto-enable.types.ts
View file

@ -22,6 +22,10 @@ export type PluginAutoEnableCandidate = { pluginId: string } & (
kind: "worker-provider-selected";
providerId: string;
}
| {
kind: "storage-provider-selected";
providerId: string;
}
| {
kind: "decision-provider-selected";
providerId: string;

44
src/config/provider-settings.ts Normal file
View file

@ -0,0 +1,44 @@
import { isPluginJsonValue } from "../plugins/host-hook-json.js";
import { isValidSecretRef } from "../secrets/ref-contract.js";
import { isSensitiveConfigPath } from "./sensitive-paths.js";
import { isSecretRef } from "./types.secrets.js";
/** Provider settings stay bounded JSON and retain secrets as references until provider use. */
export function validateProviderSettings(value: unknown, label: string): string | undefined {
if (
typeof value !== "object" ||
value === null ||
Array.isArray(value) ||
!isPluginJsonValue(value)
) {
return `${label} settings must be bounded finite JSON`;
}
const visit = (entry: unknown): string | undefined => {
if (Array.isArray(entry)) {
return entry.map(visit).find((error) => error !== undefined);
}
if (typeof entry !== "object" || entry === null) {
return undefined;
}
for (const [key, child] of Object.entries(entry)) {
const baseKey = key.replace(/ref$/i, "");
const isSensitive =
key.toLowerCase() === "keyref" ||
/^(?:accesskeyid|passphrase)$/i.test(baseKey) ||
isSensitiveConfigPath(key) ||
(baseKey !== key && isSensitiveConfigPath(baseKey));
if (isSensitive) {
if (!isSecretRef(child) || !isValidSecretRef(child)) {
return `${label} ${key} must use a SecretRef`;
}
continue;
}
const error = visit(child);
if (error) {
return error;
}
}
return undefined;
};
return visit(value);
}

2
src/config/schema.help.core.ts
View file

@ -2,6 +2,7 @@ import { META_FIELD_HELP } from "./schema.meta.js";
import { describeTalkSilenceTimeoutDefaults } from "./talk-defaults.js";
import { CLOUD_WORKER_FIELD_HELP } from "./zod-schema.cloud-workers.js";
import { DESKTOP_FIELD_HELP } from "./zod-schema.desktop.js";
import { STORAGE_FIELD_HELP } from "./zod-schema.storage.js";
import { TELEMETRY_FIELD_HELP } from "./zod-schema.telemetry.js";
export const CORE_FIELD_HELP: Record<string, string> = {
@ -82,6 +83,7 @@ export const CORE_FIELD_HELP: Record<string, string> = {
cloudWorkers:
"Opt-in cloud worker profiles for disposable remote environments. When this section is omitted or has no profiles, cloud worker creation remains unavailable and existing gateway/node status behavior is unchanged.",
...CLOUD_WORKER_FIELD_HELP,
...STORAGE_FIELD_HELP,
...DESKTOP_FIELD_HELP,
gateway:
"Gateway runtime surface for bind mode, auth, control UI, remote transport, and operational safety controls. Keep conservative defaults unless you intentionally expose the gateway beyond trusted local interfaces.",

2
src/config/schema.hints.ts
View file

@ -26,6 +26,7 @@ const GROUP_HINTS = [
["nodeHost", "Node Host", 35],
["cloudWorkers", "Cloud Workers", 37],
["desktop", "Desktop", 38],
["storage", "Storage", 39],
["agents", "Agents", 40],
["tools", "Tools", 50],
["bindings", "Bindings", 55],
@ -90,6 +91,7 @@ const SECTION_DOCS_URLS = {
voicewake: "https://docs.openclaw.ai/nodes/voicewake",
presence: "https://docs.openclaw.ai/concepts/presence",
cloudWorkers: "https://docs.openclaw.ai/gateway/cloud-workers",
storage: "https://docs.openclaw.ai/concepts/storage-locations",
desktop: "https://docs.openclaw.ai/gateway/configuration",
worktreeRoot: "https://docs.openclaw.ai/concepts/managed-worktrees",
worktreeAcceleration: "https://docs.openclaw.ai/concepts/managed-worktrees",

8
src/config/schema.labels.agent-models.ts
View file

@ -1,4 +1,12 @@
export const AGENT_MODEL_FIELD_LABELS: Record<string, string> = {
"agents.entries.*.models": "Agent Model Overrides",
"agents.entries.*.modelPolicy": "Agent Model Policy",
"agents.entries.*.modelPolicy.allow": "Allowed Agent Models",
"agents.entries.*.models.*.agentRuntime": "Agent Model Runtime",
"agents.entries.*.models.*.agentRuntime.id": "Agent Model Runtime ID",
"agents.entries.*.models.*.codeMode": "Code Mode",
"agents.entries.*.agentRuntime": "Legacy Agent Runtime",
"agents.entries.*.agentRuntime.id": "Legacy Agent Runtime ID",
"agents.defaults.models": "Models",
"agents.defaults.modelSelectionScope": "Model Selection Scope",
"agents.defaults.modelPolicy": "Model Policy",

10
src/config/schema.labels.ts
View file

@ -10,6 +10,7 @@ import { NODE_CAPABILITY_FIELD_LABELS } from "./schema.node-capabilities.js";
import { CLOUD_WORKER_FIELD_LABELS } from "./zod-schema.cloud-workers.js";
import { DESKTOP_FIELD_LABELS } from "./zod-schema.desktop.js";
import { NODE_HOST_FIELD_LABELS } from "./zod-schema.node-host.js";
import { STORAGE_FIELD_LABELS } from "./zod-schema.storage.js";
import { TELEMETRY_FIELD_LABELS } from "./zod-schema.telemetry.js";
export const FIELD_LABELS: Record<string, string> = {
@ -107,16 +108,9 @@ export const FIELD_LABELS: Record<string, string> = {
"agents.entries.*.contextLimits": "Agent Context Limits",
"agents.entries.*.contextLimits.memoryGetMaxChars": "Agent memory_get Max Chars",
"agents.entries.*.contextLimits.postCompactionMaxChars": "Agent Post-compaction Max Chars",
"agents.entries.*.models": "Agent Model Overrides",
"agents.entries.*.modelPolicy": "Agent Model Policy",
"agents.entries.*.modelPolicy.allow": "Allowed Agent Models",
"agents.entries.*.models.*.agentRuntime": "Agent Model Runtime",
"agents.entries.*.models.*.agentRuntime.id": "Agent Model Runtime ID",
"agents.entries.*.models.*.codeMode": "Code Mode",
"agents.entries.*.agentRuntime": "Legacy Agent Runtime",
"agents.entries.*.agentRuntime.id": "Legacy Agent Runtime ID",
cloudWorkers: "Cloud Workers",
...CLOUD_WORKER_FIELD_LABELS,
...STORAGE_FIELD_LABELS,
...DESKTOP_FIELD_LABELS,
...GATEWAY_FIELD_LABELS,
tools: "Tools",

2
src/config/schema.tiers.ts
View file

@ -4,7 +4,7 @@ import { asSchemaObject, type ConfigJsonSchemaObject } from "./schema.shared.js"
const ROOT_TIER_PATHS = `
accessGroups acp agents approvals attachments auth bindings broadcast browser channels
cloudWorkers commands cron desktop diagnostics discovery env gateway hooks logging mcp memory messages
meta models nodeHost plugins proxy secrets security session skills surfaces talk telemetry tools transcripts
meta models nodeHost plugins proxy secrets security session skills storage surfaces talk telemetry tools transcripts
tts ui update wizard worktreeAcceleration worktreeRoot
`
.trim()

3
src/config/types.openclaw.ts
View file

@ -22,6 +22,7 @@ import type { NodeHostConfig } from "./types.node-host.js";
import type { PluginsConfig } from "./types.plugins.js";
import type { SecretsConfig } from "./types.secrets.js";
import type { SkillsConfig } from "./types.skills.js";
import type { StorageConfig } from "./types.storage.js";
import type { TelemetryConfig } from "./types.telemetry.js";
import type { ToolsConfig } from "./types.tools.js";
import type { TtsConfig } from "./types.tts.js";
@ -130,6 +131,8 @@ export type OpenClawConfig = {
gateway?: GatewayConfig;
/** Opt-in cloud-worker provider profiles. */
cloudWorkers?: CloudWorkersConfig;
/** Named storage destinations and their encryption settings. */
storage?: StorageConfig;
/** Experimental desktop sources owned by the gateway host. */
desktop?: DesktopConfig;
/** Memory indexing/search configuration. */

5
src/config/types.storage.ts Normal file
View file

@ -0,0 +1,5 @@
import type { z } from "zod";
import type { StorageConfigSchema } from "./zod-schema.storage.js";
export type StorageConfig = NonNullable<z.input<typeof StorageConfigSchema>>;
export type StorageLocationConfig = NonNullable<StorageConfig["locations"]>[string];

45
src/config/zod-schema.cloud-workers.ts
View file

@ -1,54 +1,13 @@
// Defines cloud-worker provider profile config parsing.
import { z } from "zod";
import { parseDurationMs } from "../cli/parse-duration.js";
import { isPluginJsonValue } from "../plugins/host-hook-json.js";
import { isValidSecretRef } from "../secrets/ref-contract.js";
import { normalizeCloudRepo } from "./cloud-worker-project-profiles.js";
import { validateProviderSettings } from "./provider-settings.js";
import { projectConfigFieldMetadata } from "./schema.field-metadata.js";
import { isSensitiveConfigPath } from "./sensitive-paths.js";
import { isSecretRef } from "./types.secrets.js";
import { configUiMetadata } from "./zod-schema.sensitive.js";
export function validateCloudWorkerProfileSettings(value: unknown): string | undefined {
if (
typeof value !== "object" ||
value === null ||
Array.isArray(value) ||
!isPluginJsonValue(value)
) {
return "Worker profile settings must be bounded finite JSON";
}
const visit = (entry: unknown): string | undefined => {
if (Array.isArray(entry)) {
return entry.map(visit).find((error) => error !== undefined);
}
if (typeof entry !== "object" || entry === null) {
return undefined;
}
for (const [key, child] of Object.entries(entry)) {
const baseKey = key.replace(/ref$/i, "");
const isSensitive =
key.toLowerCase() === "keyref" ||
isSensitiveConfigPath(key) ||
(baseKey !== key && isSensitiveConfigPath(baseKey));
if (isSensitive) {
if (!isSecretRef(child) || !isValidSecretRef(child)) {
return `Worker profile ${key} must use a SecretRef`;
}
continue;
}
const error = visit(child);
if (error) {
return error;
}
}
return undefined;
};
return visit(value);
}
const CloudWorkerSettingsSchema = z.record(z.string(), z.unknown()).superRefine((value, ctx) => {
const message = validateCloudWorkerProfileSettings(value);
const message = validateProviderSettings(value, "Worker profile");
if (message) {
ctx.addIssue({ code: "custom", message });
}

2
src/config/zod-schema.root-shape.ts
View file

@ -34,6 +34,7 @@ import {
} from "./zod-schema.root-support.js";
import { sensitive } from "./zod-schema.sensitive.js";
import { CommandsSchema, MessagesSchema, SessionSchema } from "./zod-schema.session.js";
import { StorageConfigSchema } from "./zod-schema.storage.js";
import { TelemetryConfigSchema } from "./zod-schema.telemetry.js";
export const OpenClawSchemaShape = {
@ -504,6 +505,7 @@ export const OpenClawSchemaShape = {
talk: TalkSchema.optional(),
gateway: GatewayConfigSchema,
cloudWorkers: CloudWorkersConfigSchema,
storage: StorageConfigSchema,
desktop: DesktopConfigSchema,
memory: MemorySchema,
mcp: McpConfigSchema,

86
src/config/zod-schema.storage.test.ts Normal file
View file

@ -0,0 +1,86 @@
import { describe, expect, it } from "vitest";
import { OpenClawSchema } from "./zod-schema.js";
const passphrase = { source: "env", provider: "default", id: "STORAGE_PASSPHRASE" };
function location(overrides: Record<string, unknown> = {}, name = "archive") {
return {
storage: {
locations: {
[name]: {
provider: "filesystem",
settings: { path: "/mnt/archive/openclaw" },
encryption: { passphrase },
...overrides,
},
},
},
};
}
describe("OpenClawSchema storage config", () => {
it.each([{ passphrase }, "none"])("preserves an explicit encryption choice: %j", (encryption) => {
const config = location({ encryption });
expect(OpenClawSchema.parse(config).storage).toEqual(config.storage);
});
it.each([undefined, {}, "aes256", { passphrase: 123 }])(
"rejects missing or invalid encryption: %j",
(encryption) => {
const result = OpenClawSchema.safeParse(location({ encryption }));
expect(result.success).toBe(false);
if (!result.success) {
expect(result.error.issues[0]?.path).toEqual([
"storage",
"locations",
"archive",
"encryption",
]);
}
},
);
it.each(["", "Archive", "-archive", "archive_disk", "a".repeat(64)])(
"rejects an invalid location name: %s",
(name) => {
expect(OpenClawSchema.safeParse(location({}, name)).success).toBe(false);
},
);
it("accepts provider settings with nested secret references", () => {
const settings = {
accountId: "example-account",
bucket: "example-bucket",
accessKeyId: { source: "env", provider: "default", id: "STORAGE_ACCESS_KEY_ID" },
auth: [{ secretAccessKey: { source: "env", provider: "default", id: "STORAGE_SECRET" } }],
};
const config = location({ provider: "example-provider", settings });
expect(OpenClawSchema.parse(config).storage).toEqual(config.storage);
});
it.each([
{ accessKeyId: "example-key-not-real" },
{ auth: [{ secretAccessKey: "example-secret-not-real" }] },
{ auth: { passphrase: "example-passphrase-not-real" } },
{ keyRef: { source: "env", provider: "default", id: "invalid-id" } },
])("rejects plaintext or malformed provider credentials: %j", (settings) => {
const result = OpenClawSchema.safeParse(location({ settings }));
expect(result.success).toBe(false);
if (!result.success) {
expect(result.error.issues[0]?.message).toContain("must use a SecretRef");
}
});
it.each([{ timeout: Infinity }, { path: "a".repeat(65_537) }])(
"rejects nonfinite or oversized provider settings %#",
(settings) => {
const result = OpenClawSchema.safeParse(location({ settings }));
expect(result.success).toBe(false);
if (!result.success) {
expect(result.error.issues[0]?.message).toBe(
"Storage location settings must be bounded finite JSON",
);
}
},
);
});

61
src/config/zod-schema.storage.ts Normal file
View file

@ -0,0 +1,61 @@
import { z } from "zod";
import { validateProviderSettings } from "./provider-settings.js";
import { projectConfigFieldMetadata } from "./schema.field-metadata.js";
import { SecretInputSchema } from "./zod-schema.secret-input.js";
import { configUiMetadata, sensitive } from "./zod-schema.sensitive.js";
const StorageSettingsSchema = z.record(z.string(), z.unknown()).superRefine((value, ctx) => {
const message = validateProviderSettings(value, "Storage location");
if (message) {
ctx.addIssue({ code: "custom", message });
}
});
const StorageLocationSchema = z
.strictObject({
provider: z.string().trim().min(1).register(configUiMetadata, {
label: "Storage Provider",
help: 'Storage provider id, such as "filesystem". Referencing a bundled provider automatically enables its plugin unless plugin policy disables it.',
}),
settings: StorageSettingsSchema.register(configUiMetadata, {
label: "Storage Provider Settings",
help: "Provider-owned bounded JSON settings. Secret-bearing values must use SecretRef objects, which the provider resolves only when needed. The filesystem provider requires an absolute path to an existing directory.",
}),
encryption: z
.union([
z.literal("none"),
z.strictObject({
passphrase: SecretInputSchema.register(sensitive).register(configUiMetadata, {
label: "Storage Encryption Passphrase",
help: "Passphrase used to encrypt this location. Prefer a SecretRef and keep a recoverable copy: losing or changing the passphrase makes existing encrypted objects unreadable.",
}),
}),
])
.register(configUiMetadata, {
label: "Storage Encryption",
help: 'Required encryption choice: a passphrase object encrypts objects before upload; "none" explicitly disables encryption. Backups can contain credentials, so use "none" only when the destination already provides suitable protection.',
}),
})
.register(configUiMetadata, {
label: "Storage Location",
help: "One named storage destination with provider settings and an explicit encryption choice. Initialize a new destination with openclaw storage init before writing objects.",
});
export const StorageConfigSchema = z
.strictObject({
locations: z
.record(z.string().regex(/^[a-z0-9][a-z0-9-]{0,62}$/), StorageLocationSchema)
.optional()
.register(configUiMetadata, {
label: "Storage Locations",
help: "Named storage destinations. Names contain 1–63 lowercase letters, digits, or hyphens and start with a letter or digit. Each destination keeps an initialization marker that runtime writes never create.",
}),
})
.optional()
.register(configUiMetadata, {
label: "Storage",
help: "Reusable storage locations for OpenClaw artifacts. Core storage owns location identity, encryption, and health; providers transport objects and consumers decide what to retain.",
});
export const { labels: STORAGE_FIELD_LABELS, help: STORAGE_FIELD_HELP } =
projectConfigFieldMetadata(StorageConfigSchema, "storage");

207
src/cron/backup-command.ts
View file

@ -1,16 +1,211 @@
import type { CronJob } from "./types.js";
import { parseArgs } from "node:util";
import {
normalizeBackupRetention,
resolveBackupNamespace,
type BackupRetention,
} from "../infra/backup-retention.js";
import type { CronJob, CronJobCreate } from "./types.js";
export const SCHEDULED_BACKUP_DECLARATION_KEY = "openclaw-backup-scheduled";
export const SCHEDULED_BACKUP_COMMAND = ["openclaw", "backup", "git", "create"] as const;
// Installed Git schedules retain this declaration and command contract.
const BACKUP_SCHEDULE_DECLARATION_KEYS = {
git: "openclaw-backup-scheduled",
offsite: "openclaw-backup-offsite-scheduled",
} as const;
/** Identifies the command contract emitted by backup enable, not its display label. */
const BACKUP_COMMANDS = {
git: ["openclaw", "backup", "git", "create"],
offsite: ["openclaw", "backup", "create"],
} as const;
export type BackupScheduleSpec = { everyMs: number } & (
| {
mode: "git";
repository: string;
scope: { kind: "all" | "global" } | { kind: "agent"; agentId: string };
push: boolean;
excludeSecrets: boolean;
}
| (BackupRetention & {
mode: "offsite";
location: string;
namespace: string;
claimNamespace?: boolean;
includeWorkspace: boolean;
})
);
export type BackupScheduleSummary = {
id: string;
mode: BackupScheduleSpec["mode"];
target: string;
namespace?: string;
enabled: boolean;
everyMs: number;
nextRunAtMs?: number;
};
export function backupScheduleModeForDeclaration(
declarationKey: string | undefined,
): BackupScheduleSpec["mode"] | undefined {
if (declarationKey === BACKUP_SCHEDULE_DECLARATION_KEYS.git) {
return "git";
}
if (declarationKey === BACKUP_SCHEDULE_DECLARATION_KEYS.offsite) {
return "offsite";
}
return undefined;
}
/** Identifies managed command execution even when an operator edits its arguments. */
export function isScheduledBackupCommand(
job: Pick<CronJob, "declarationKey" | "payload">,
): boolean {
const mode = backupScheduleModeForDeclaration(job.declarationKey);
const payload = job.payload;
return (
job.declarationKey === SCHEDULED_BACKUP_DECLARATION_KEY &&
mode !== undefined &&
payload.kind === "command" &&
SCHEDULED_BACKUP_COMMAND.every((part, index) => payload.argv[index] === part)
BACKUP_COMMANDS[mode].every((part, index) => payload.argv[index] === part)
);
}
export function buildBackupScheduleJob(spec: BackupScheduleSpec): CronJobCreate {
const argv: string[] = [...BACKUP_COMMANDS[spec.mode]];
if (spec.mode === "git") {
argv.push("--repository", spec.repository);
argv.push(
...(spec.scope.kind === "agent" ? ["--agent", spec.scope.agentId] : [`--${spec.scope.kind}`]),
);
if (spec.push) {
argv.push("--push");
}
if (spec.excludeSecrets) {
argv.push("--exclude-secrets");
}
} else {
argv.push("--to", spec.location, "--namespace", spec.namespace);
if (spec.claimNamespace) {
argv.push("--claim-namespace");
}
if (!spec.includeWorkspace) {
argv.push("--no-include-workspace");
}
for (const [flag, value] of [
["--keep-daily", spec.keepDaily],
["--keep-weekly", spec.keepWeekly],
["--keep-monthly", spec.keepMonthly],
] as const) {
if (value !== undefined) {
argv.push(flag, String(value));
}
}
}
const declarationKey = BACKUP_SCHEDULE_DECLARATION_KEYS[spec.mode];
return {
declarationKey,
name: declarationKey,
enabled: true,
schedule: { kind: "every", everyMs: spec.everyMs },
sessionTarget: "isolated",
wakeMode: "now",
payload: { kind: "command", argv },
delivery: { mode: "none" },
};
}
/** Decode the persisted argv contract for status without relying on display names. */
function parseBackupScheduleJob(
job: Pick<CronJob, "declarationKey" | "payload" | "schedule">,
): BackupScheduleSpec | undefined {
const mode = backupScheduleModeForDeclaration(job.declarationKey);
if (
!mode ||
!isScheduledBackupCommand(job) ||
job.payload.kind !== "command" ||
job.schedule.kind !== "every"
) {
return undefined;
}
const everyMs = job.schedule.everyMs;
try {
const args = job.payload.argv.slice(BACKUP_COMMANDS[mode].length);
if (mode === "git") {
const { values } = parseArgs({
args,
options: {
repository: { type: "string" },
all: { type: "boolean" },
global: { type: "boolean" },
agent: { type: "string" },
push: { type: "boolean" },
"exclude-secrets": { type: "boolean" },
},
});
if (
!values.repository ||
[values.all, values.global, values.agent].filter(Boolean).length > 1
) {
return undefined;
}
return {
mode,
everyMs,
repository: values.repository,
scope: values.agent
? { kind: "agent", agentId: values.agent }
: { kind: values.global ? "global" : "all" },
push: values.push === true,
excludeSecrets: values["exclude-secrets"] === true,
};
}
const { values } = parseArgs({
args,
options: {
to: { type: "string" },
namespace: { type: "string" },
"claim-namespace": { type: "boolean" },
"no-include-workspace": { type: "boolean" },
"keep-daily": { type: "string" },
"keep-weekly": { type: "string" },
"keep-monthly": { type: "string" },
},
});
if (!values.to) {
return undefined;
}
return {
mode,
everyMs,
location: values.to,
namespace: resolveBackupNamespace(values.namespace),
claimNamespace: values["claim-namespace"] === true,
includeWorkspace: values["no-include-workspace"] !== true,
...normalizeBackupRetention({
keepDaily: values["keep-daily"],
keepWeekly: values["keep-weekly"],
keepMonthly: values["keep-monthly"],
}),
};
} catch {
return undefined;
}
}
export function summarizeBackupSchedules(jobs: readonly CronJob[]): BackupScheduleSummary[] {
return jobs.flatMap((job) => {
const spec = parseBackupScheduleJob(job);
return spec
? [
{
id: job.id,
mode: spec.mode,
target: spec.mode === "git" ? spec.repository : spec.location,
...(spec.mode === "offsite" ? { namespace: spec.namespace } : {}),
enabled: job.enabled,
everyMs: spec.everyMs,
...(job.state.nextRunAtMs === undefined ? {} : { nextRunAtMs: job.state.nextRunAtMs }),
},
]
: [];
});
}

18
src/cron/command-runner.test.ts
View file

@ -19,10 +19,12 @@ import { resetGatewayWorkAdmission } from "../process/gateway-work-admission.js"
import { isPidAlive } from "../shared/pid-alive.js";
import { readPidFile, waitForPidToExit } from "../test-utils/process-tree.js";
import { withTempDir } from "../test-utils/temp-dir.js";
import { SCHEDULED_BACKUP_COMMAND, SCHEDULED_BACKUP_DECLARATION_KEY } from "./backup-command.js";
import { runCronCommandJob } from "./command-runner.js";
import type { CronJob } from "./types.js";
const SCHEDULED_BACKUP_COMMAND = ["openclaw", "backup", "git", "create"];
const SCHEDULED_BACKUP_DECLARATION_KEY = "openclaw-backup-scheduled";
function makeCommandJob(payload: Extract<CronJob["payload"], { kind: "command" }>): CronJob {
const now = Date.now();
return {
@ -40,7 +42,7 @@ function makeCommandJob(payload: Extract<CronJob["payload"], { kind: "command" }
}
describe("runCronCommandJob", () => {
it.each(["owned", "display-only", "retargeted"])(
it.each(["owned", "offsite", "display-only", "retargeted"])(
"records only declared backup command custody through native settlement: %s",
async (mode) => {
const settled = createDeferred<SpawnResult>();
@ -49,11 +51,19 @@ describe("runCronCommandJob", () => {
.mockReturnValue(settled.promise);
const job = makeCommandJob({
kind: "command",
argv: mode === "retargeted" ? ["echo", "backup"] : [...SCHEDULED_BACKUP_COMMAND],
argv:
mode === "retargeted"
? ["echo", "backup"]
: mode === "offsite"
? ["openclaw", "backup", "create", "--to", "archive"]
: [...SCHEDULED_BACKUP_COMMAND],
});
job.name = SCHEDULED_BACKUP_DECLARATION_KEY;
if (mode !== "display-only") {
job.declarationKey = SCHEDULED_BACKUP_DECLARATION_KEY;
job.declarationKey =
mode === "offsite"
? "openclaw-backup-offsite-scheduled"
: SCHEDULED_BACKUP_DECLARATION_KEY;
}
const ownsBackup = mode !== "display-only" && mode !== "retargeted";
const running = runCronCommandJob({ job });

2
src/flows/doctor-health-contribution-runners.state.ts
View file

@ -126,7 +126,7 @@ export async function runStateIntegrityHealth(ctx: DoctorHealthFlowContext): Pro
await noteStateIntegrity(ctx.cfg, ctx.prompter, ctx.configPath, {
stateDirExistedAtStart: ctx.stateDirExistedAtStart,
});
await noteBackupDoctorHint(ctx.env ?? process.env);
await noteBackupDoctorHint(ctx.env ?? process.env, ctx.cfg);
const { noteBackupScratchHealth } = await import("../commands/doctor-backup-scratch.js");
await noteBackupScratchHealth(ctx.env ?? process.env, ctx.prompter.shouldRepair);
}

3
src/gateway/methods/core-descriptors.ts
View file

@ -686,4 +686,7 @@ export const CORE_GATEWAY_METHOD_SPECS = [
["gateway.stop.request", "restart", "operator.admin", "2026.9", CONTROL_PLANE_WRITE],
["diagnostics.heapSnapshot", "diagnostics", "operator.admin", "2026.9"],
["sessions.catalog.import", "session-catalog", "operator.write", "2026.9"],
["backup.status", "backup", "operator.read", "2026.9"],
["storage.locations.list", "storage", "operator.read", "2026.9"],
["storage.locations.probe", "storage", "operator.read", "2026.9"],
] as const satisfies readonly CoreGatewayMethodSpecRow[];

9
src/gateway/server-methods-list.test.ts
View file

@ -251,6 +251,9 @@ describe("listGatewayMethods", () => {
"gateway.stop.request",
"diagnostics.heapSnapshot",
"sessions.catalog.import",
"backup.status",
"storage.locations.list",
"storage.locations.probe",
];
expect(listGatewayMethods().slice(-expectedSuffix.length)).toEqual(expectedSuffix);
const methods = listGatewayMethods();
@ -323,6 +326,9 @@ describe("listGatewayMethods", () => {
"gateway.stop.request",
"diagnostics.heapSnapshot",
"sessions.catalog.import",
"backup.status",
"storage.locations.list",
"storage.locations.probe",
]);
});
@ -523,6 +529,9 @@ describe("listGatewayMethods", () => {
"gateway.stop.request",
"diagnostics.heapSnapshot",
"sessions.catalog.import",
"backup.status",
"storage.locations.list",
"storage.locations.probe",
];
expect(coreMethods.slice(-expectedCoreSuffix.length)).toEqual(expectedCoreSuffix);
expect(methods.indexOf("approval.get")).toBeGreaterThan(methods.indexOf("tts.speak"));

188
src/gateway/server-methods/backup.test.ts Normal file
View file

@ -0,0 +1,188 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { buildBackupScheduleJob } from "../../cron/backup-command.js";
import { CronService } from "../../cron/service.js";
import { createNoopLogger } from "../../cron/service.test-harness.js";
import type { CronJob } from "../../cron/types.js";
import type { BackupRunRecord } from "../../state/backup-run-records.js";
import { createTestGatewayScheduler } from "../../test-utils/gateway-scheduler-clock.js";
import { authorizeOperatorScopesForMethod } from "../method-scopes.js";
import { createDirectChatContext } from "../server-chat.agent-events.test-helpers.js";
import { coreGatewayHandlers } from "./core-handlers.js";
import type { RespondFn } from "./types.js";
const ledger = vi.hoisted(() => vi.fn<() => Promise<BackupRunRecord[]>>());
vi.mock("../../state/backup-run-records.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../../state/backup-run-records.js")>()),
readBackupRuns: ledger,
}));
vi.mock("../../plugins/active-runtime-registry.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../../plugins/active-runtime-registry.js")>()),
getLoadedRuntimePluginRegistry: () => undefined,
}));
afterEach(() => {
vi.restoreAllMocks();
ledger.mockReset();
});
async function invoke(params: Record<string, unknown>, jobs: CronJob[] = []) {
const cron = new CronService({
scheduler: createTestGatewayScheduler(),
storePath: "/unused/backup-rpc/jobs.json",
cronEnabled: false,
defaultAgentId: "main",
log: createNoopLogger(),
enqueueSystemEvent: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
vi.spyOn(cron, "list").mockResolvedValue(jobs);
const context = createDirectChatContext({
cron,
getRuntimeConfig: () => ({
storage: {
locations: {
offsite: {
provider: "filesystem",
settings: { path: "/missing/remote" },
encryption: "none",
},
},
},
}),
});
const respond = vi.fn<RespondFn>();
const handler = coreGatewayHandlers["backup.status"];
if (!handler) {
throw new Error("Missing backup.status handler");
}
await handler({
req: { type: "req", id: "backup", method: "backup.status" },
params,
context,
client: null,
isWebchatConnect: () => false,
respond,
});
return respond;
}
describe("backup.status", () => {
it("returns destination freshness, cron-owned schedules and a config-only storage view", async () => {
const ok: BackupRunRecord = {
id: "ok",
createdAt: 10,
archivePath: "",
kind: "archive",
status: "ok",
target: "offsite",
namespace: "host",
bytes: 12,
};
const failed: BackupRunRecord = {
...ok,
id: "failed",
createdAt: 20,
status: "failed",
error: "disk unavailable",
};
const archiveOk: BackupRunRecord = {
id: "archive-ok",
createdAt: 10,
archivePath: "/backups/old.tar.gz",
kind: "archive",
status: "ok",
};
const archiveFailed: BackupRunRecord = {
...archiveOk,
id: "archive-failed",
createdAt: 20,
archivePath: "/backups/new.tar.gz",
status: "failed",
};
const snapshotOk: BackupRunRecord = {
...archiveOk,
id: "snapshot-ok",
kind: "sqlite-snapshot",
archivePath: "/backups/old-snapshot",
};
const snapshotFailed: BackupRunRecord = {
...snapshotOk,
id: "snapshot-failed",
createdAt: 20,
archivePath: "/backups/new-snapshot",
status: "failed",
};
ledger.mockResolvedValue([failed, archiveFailed, snapshotFailed, ok, archiveOk, snapshotOk]);
const job: CronJob = {
...buildBackupScheduleJob({
mode: "offsite",
location: "offsite",
namespace: "host",
includeWorkspace: true,
everyMs: 60_000,
}),
id: "backup-job",
enabled: true,
createdAtMs: 1,
updatedAtMs: 1,
state: { nextRunAtMs: 70_000 },
};
const respond = await invoke({}, [job]);
expect(respond).toHaveBeenCalledWith(
true,
{
targets: [
{ kind: "archive", target: "offsite", namespace: "host", latest: failed, latestOk: ok },
{
kind: "archive",
target: "/backups/new.tar.gz",
latest: archiveFailed,
latestOk: archiveOk,
},
{
kind: "sqlite-snapshot",
target: "/backups/new-snapshot",
latest: snapshotFailed,
latestOk: snapshotOk,
},
],
schedules: [
{
id: "backup-job",
mode: "offsite",
target: "offsite",
namespace: "host",
enabled: true,
everyMs: 60_000,
nextRunAtMs: 70_000,
},
],
locations: [
{
name: "offsite",
provider: "filesystem",
displayTarget: "/missing/remote",
encrypted: false,
},
],
},
undefined,
);
});
it("requires operator read scope and rejects unsupported parameters before reading", async () => {
expect(authorizeOperatorScopesForMethod("backup.status", ["operator.read"])).toEqual({
allowed: true,
});
expect(authorizeOperatorScopesForMethod("backup.status", [])).toEqual({
allowed: false,
missingScope: "operator.read",
});
expect(await invoke({ unexpected: true })).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({ code: "INVALID_REQUEST" }),
);
expect(ledger).not.toHaveBeenCalled();
});
});

32
src/gateway/server-methods/backup.ts Normal file
View file

@ -0,0 +1,32 @@
import {
type BackupStatusResult,
validateBackupStatusParams,
} from "../../../packages/gateway-protocol/src/index.js";
import { summarizeBackupSchedules } from "../../cron/backup-command.js";
import { getLoadedRuntimePluginRegistry } from "../../plugins/active-runtime-registry.js";
import { readBackupRuns, summarizeBackupTargets } from "../../state/backup-run-records.js";
import { listStorageLocations } from "../../storage/locations.js";
import type { GatewayRequestHandlers } from "./types.js";
import { defineValidatedGatewayMethod } from "./validation.js";
export const backupHandlers: GatewayRequestHandlers = {
"backup.status": defineValidatedGatewayMethod(
"backup.status",
validateBackupStatusParams,
async ({ context, respond }) => {
const [runs, jobs] = await Promise.all([
readBackupRuns(process.env),
context.cron.list({ includeDisabled: true }),
]);
const result: BackupStatusResult = {
targets: summarizeBackupTargets(runs),
schedules: summarizeBackupSchedules(jobs),
locations: listStorageLocations(
context.getRuntimeConfig(),
getLoadedRuntimePluginRegistry() ?? undefined,
),
};
respond(true, result, undefined);
},
),
};

2
src/gateway/server-methods/core-handlers.ts
View file

@ -94,6 +94,8 @@ const CORE_GATEWAY_HANDLER_MODULES = {
portals: () => import("./portals.js").then((module) => module.portalHandlers),
"progress-card": () => import("./progress-card.js").then((module) => module.progressCardHandlers),
migrations: () => import("./migrations.js").then((module) => module.migrationsHandlers),
backup: () => import("./backup.js").then((module) => module.backupHandlers),
storage: () => import("./storage.js").then((module) => module.storageHandlers),
push: () => import("./push.js").then((module) => module.pushHandlers),
restart: () => import("./restart.js").then((module) => module.restartHandlers),
suspend: () => import("./suspend.js").then((module) => module.suspendHandlers),

188
src/gateway/server-methods/storage.test.ts Normal file
View file

@ -0,0 +1,188 @@
import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import type { StorageRegistry } from "../../storage/provider.js";
import { authorizeOperatorScopesForMethod } from "../method-scopes.js";
import { createDirectChatContext } from "../server-chat.agent-events.test-helpers.js";
import { coreGatewayHandlers } from "./core-handlers.js";
import type { RespondFn } from "./types.js";
const pluginInspection = vi.hoisted(() => vi.fn());
const loadedRegistry = vi.hoisted(() => vi.fn<() => StorageRegistry | undefined>());
vi.mock("../../plugins/active-runtime-registry.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../../plugins/active-runtime-registry.js")>()),
getLoadedRuntimePluginRegistry: loadedRegistry,
}));
vi.mock("../../plugins/loader.js", () => ({
acquirePluginRegistryForInspection: pluginInspection,
}));
vi.mock("../../plugins/manifest-contract-runtime.js", () => ({
resolveManifestContractRuntimePluginResolution: () => ({ pluginIds: ["fixture-storage"] }),
}));
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
afterEach(() => loadedRegistry.mockReset());
async function invoke(
method: "storage.locations.list" | "storage.locations.probe",
config: OpenClawConfig,
params: Record<string, unknown>,
) {
const respond = vi.fn<RespondFn>();
const handler = coreGatewayHandlers[method];
if (!handler) {
throw new Error(`Missing ${method} handler`);
}
await handler({
req: { type: "req", id: "storage-request", method },
params,
context: createDirectChatContext({ getRuntimeConfig: () => config }),
client: null,
isWebchatConnect: () => false,
respond,
});
return respond;
}
describe("storage Gateway methods", () => {
it("describes loaded providers without opening or activating providers", async () => {
const open = vi.fn();
const storageProviders: StorageRegistry["storageProviders"] = new Map();
storageProviders.set("memory", {
pluginId: "fixture-storage",
source: "test",
provider: {
id: "memory",
label: "Memory",
open,
describeTarget: (settings) =>
typeof settings.bucket === "string" ? `memory://${settings.bucket}` : undefined,
},
});
storageProviders.set("opaque", {
pluginId: "fixture-storage",
source: "test",
provider: { id: "opaque", label: "Opaque", open },
});
loadedRegistry.mockReturnValue({ storageProviders });
const respond = await invoke(
"storage.locations.list",
{
storage: {
locations: {
archive: { provider: "memory", settings: { bucket: "example" }, encryption: "none" },
opaque: { provider: "opaque", settings: {}, encryption: "none" },
unavailable: { provider: "unloaded", settings: {}, encryption: "none" },
},
},
},
{},
);
expect(respond).toHaveBeenCalledWith(
true,
{
locations: [
{
name: "archive",
provider: "memory",
displayTarget: "memory://example",
encrypted: false,
},
{ name: "opaque", provider: "opaque", encrypted: false },
{ name: "unavailable", provider: "unloaded", encrypted: false },
],
},
undefined,
);
expect(open).not.toHaveBeenCalled();
expect(pluginInspection).not.toHaveBeenCalled();
});
it("does not activate an unavailable provider through CLI plugin inspection", async () => {
const respond = await invoke(
"storage.locations.probe",
{
storage: {
locations: { remote: { provider: "fixture-storage", settings: {}, encryption: "none" } },
},
},
{ name: "remote" },
);
expect(respond).toHaveBeenCalledWith(
true,
expect.objectContaining({ state: "error" }),
undefined,
);
expect(pluginInspection).not.toHaveBeenCalled();
});
it("lists config without opening a missing destination or exposing settings and secrets", async () => {
const root = path.join(tempDirs.make("openclaw-storage-rpc-"), "missing");
const config: OpenClawConfig = {
storage: {
locations: {
archive: {
provider: "filesystem",
settings: { path: root },
encryption: {
passphrase: { source: "env", provider: "default", id: "STORAGE_TEST_KEY" },
},
},
},
},
};
const respond = await invoke("storage.locations.list", config, {});
expect(respond).toHaveBeenCalledWith(
true,
{
locations: [
{ name: "archive", provider: "filesystem", displayTarget: root, encrypted: true },
],
},
undefined,
);
await expect(fs.stat(root)).rejects.toMatchObject({ code: "ENOENT" });
});
it("reports an unavailable destination through probe without creating its root", async () => {
const root = path.join(tempDirs.make("openclaw-storage-rpc-"), "missing");
const respond = await invoke(
"storage.locations.probe",
{
storage: {
locations: {
archive: { provider: "filesystem", settings: { path: root }, encryption: "none" },
},
},
},
{ name: "archive" },
);
expect(respond).toHaveBeenCalledWith(
true,
expect.objectContaining({ state: "unavailable" }),
undefined,
);
await expect(fs.stat(root)).rejects.toMatchObject({ code: "ENOENT" });
});
it.each(["storage.locations.list", "storage.locations.probe"] as const)(
"%s is available with read scope and rejects unknown request properties",
async (method) => {
expect(authorizeOperatorScopesForMethod(method, ["operator.read"])).toEqual({
allowed: true,
});
expect(authorizeOperatorScopesForMethod(method, [])).toEqual({
allowed: false,
missingScope: "operator.read",
});
const respond = await invoke(method, {}, { name: "../invalid", unexpected: true });
expect(respond).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({ code: "INVALID_REQUEST" }),
);
},
);
});

38
src/gateway/server-methods/storage.ts Normal file
View file

@ -0,0 +1,38 @@
import {
type StorageLocationsListResult,
type StorageLocationsProbeResult,
validateStorageLocationsListParams,
validateStorageLocationsProbeParams,
} from "../../../packages/gateway-protocol/src/index.js";
import { getLoadedRuntimePluginRegistry } from "../../plugins/active-runtime-registry.js";
import { listStorageLocations, probeStorageLocation } from "../../storage/locations.js";
import type { GatewayRequestHandlers } from "./types.js";
import { defineValidatedGatewayMethod } from "./validation.js";
export const storageHandlers: GatewayRequestHandlers = {
"storage.locations.list": defineValidatedGatewayMethod(
"storage.locations.list",
validateStorageLocationsListParams,
({ context, respond }) => {
const result: StorageLocationsListResult = {
locations: listStorageLocations(
context.getRuntimeConfig(),
getLoadedRuntimePluginRegistry() ?? undefined,
),
};
respond(true, result, undefined);
},
),
"storage.locations.probe": defineValidatedGatewayMethod(
"storage.locations.probe",
validateStorageLocationsProbeParams,
async ({ params, context, respond }) => {
const result: StorageLocationsProbeResult = await probeStorageLocation({
name: params.name,
config: context.getRuntimeConfig(),
registry: getLoadedRuntimePluginRegistry() ?? { storageProviders: new Map() },
});
respond(true, result, undefined);
},
),
};

4
src/gateway/worker-environments/provider-intent.test.ts
View file

@ -3,7 +3,7 @@ import path from "node:path";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import { describe, expect, it, vi } from "vitest";
import { requireGit } from "../../agents/worktrees/git.js";
import { validateCloudWorkerProfileSettings } from "../../config/zod-schema.cloud-workers.js";
import { validateProviderSettings } from "../../config/provider-settings.js";
import type { WorkerProvider } from "../../plugins/types.js";
import { createDeferredCore } from "../../shared/deferred.js";
import { readWorkerProjectPreparation } from "./preparation-identity.js";
@ -71,7 +71,7 @@ describe("prepared worker intent admission", () => {
projectNamespace: "gateway-test",
providerFor: () => provider,
requireWorkerProfile: (value) => {
const error = validateCloudWorkerProfileSettings(value);
const error = validateProviderSettings(value, "Worker profile");
if (error) {
throw new Error(error);
}

4
src/gateway/worker-environments/service-validation.ts
View file

@ -5,7 +5,7 @@ import {
WorkerMachineOptionsSchema,
WorkerOperatingSystemSchema,
} from "../../../packages/gateway-protocol/src/schema/environments.js";
import { validateCloudWorkerProfileSettings } from "../../config/zod-schema.cloud-workers.js";
import { validateProviderSettings } from "../../config/provider-settings.js";
import { normalizeCapabilityProviderId } from "../../plugins/provider-registry-shared.js";
import {
WorkerProviderError,
@ -25,7 +25,7 @@ export function requireWorkerProfile(
value: unknown,
serviceError: (code: "invalid_profile", message: string) => Error,
): WorkerProfile {
const error = validateCloudWorkerProfileSettings(value);
const error = validateProviderSettings(value, "Worker profile");
if (error) {
throw serviceError("invalid_profile", error);
}

119
src/infra/backup-retention.test.ts Normal file
View file

@ -0,0 +1,119 @@
import os from "node:os";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
normalizeBackupRetention,
resolveBackupNamespace,
selectBackupRetention,
type BackupRetention,
} from "./backup-retention.js";
const key = (timestamp: string) => `${timestamp}-0123abcd.tar.gz`;
const timestamps = [
"20270104T000100Z",
"20270104T000000Z",
"20270103T235900Z",
"20270101T230000Z",
"20261231T235900Z",
"20261228T010000Z",
"20261227T235900Z",
"20261201T120000Z",
"20261130T120000Z",
"20261101T120000Z",
];
const backups = timestamps.map(key);
afterEach(() => vi.restoreAllMocks());
describe("backup retention", () => {
it.each<{ label: string; policy: BackupRetention; kept: string[] }>([
{ label: "no deletion without flags", policy: {}, kept: timestamps },
{
label: "newest even when every count is zero",
policy: { keepDaily: 0, keepWeekly: 0, keepMonthly: 0 },
kept: ["20270104T000100Z"],
},
{
label: "newest in each nonempty UTC day",
policy: { keepDaily: 3 },
kept: ["20270104T000100Z", "20270103T235900Z", "20270101T230000Z"],
},
{
label: "Monday weeks across a year boundary",
policy: { keepWeekly: 2 },
kept: ["20270104T000100Z", "20270103T235900Z"],
},
{
label: "UTC calendar months across a year boundary",
policy: { keepMonthly: 2 },
kept: ["20270104T000100Z", "20261231T235900Z"],
},
{
label: "union of daily, weekly, and monthly buckets",
policy: { keepDaily: 2, keepWeekly: 3, keepMonthly: 3 },
kept: [
"20270104T000100Z",
"20270103T235900Z",
"20261231T235900Z",
"20261227T235900Z",
"20261130T120000Z",
],
},
])("keeps $label", ({ policy, kept }) => {
const selected = selectBackupRetention(backups.toReversed(), policy);
expect(selected.kept).toEqual(kept.map(key));
expect(selected.deleted).toEqual(
timestamps.filter((timestamp) => !kept.includes(timestamp)).map(key),
);
});
it.each([{}, { keepDaily: 0 }])("never selects foreign or malformed keys with %j", (policy) => {
const foreign = [
`other-host/${backups[0]}`,
`backups/other-host/${backups[0]}`,
"notes.txt",
"archive.tar.gz",
"20270101T000000Z-deadbeef.tar.gz.extra",
"20270101T000000Z-deadbee.tar.gz",
"20260230T000000Z-0123abcd.tar.gz",
"20270101T250000Z-0123abcd.tar.gz",
];
const result = selectBackupRetention([...foreign, ...backups], policy);
expect([...result.kept, ...result.deleted].toSorted()).toEqual(backups.toSorted());
expect(result.kept[0]).toBe(backups[0]);
});
it("accepts zero and integer CLI counts", () => {
expect(normalizeBackupRetention({ keepDaily: "7", keepWeekly: 0, keepMonthly: "12" })).toEqual({
keepDaily: 7,
keepWeekly: 0,
keepMonthly: 12,
});
});
it.each([
{ keepDaily: "" },
{ keepDaily: "-1" },
{ keepDaily: -1 },
{ keepWeekly: "1.5" },
{ keepMonthly: Infinity },
{ keepMonthly: Number.NaN },
{ keepDaily: Number.MAX_SAFE_INTEGER + 1 },
])("rejects unsafe retention counts %j", (options) => {
expect(() => normalizeBackupRetention(options)).toThrow("must be a nonnegative integer");
});
});
describe("backup namespaces", () => {
it("sanitizes the default hostname and preserves explicit namespace identity", () => {
vi.spyOn(os, "hostname").mockReturnValue("Peter's Mac.local");
expect(resolveBackupNamespace()).toBe("Peter-s-Mac.local");
expect(resolveBackupNamespace("Host_A-2.local")).toBe("Host_A-2.local");
});
it.each(["", ".", "..", "../other-host", "host/child", "host name", "a".repeat(129)])(
"rejects non-segment namespace %j",
(namespace) => {
expect(() => resolveBackupNamespace(namespace)).toThrow("Backup namespace");
},
);
});

106
src/infra/backup-retention.ts Normal file
View file

@ -0,0 +1,106 @@
import { randomBytes } from "node:crypto";
import os from "node:os";
export type BackupRetention = {
keepDaily?: number;
keepWeekly?: number;
keepMonthly?: number;
};
export type BackupRetentionOptions = {
[K in keyof BackupRetention]?: string | number;
};
export function normalizeBackupRetention(options: BackupRetentionOptions): BackupRetention {
const result: BackupRetention = {};
for (const key of ["keepDaily", "keepWeekly", "keepMonthly"] as const) {
const value = options[key];
if (value === undefined) {
continue;
}
const count = typeof value === "string" && /^\d+$/.test(value) ? Number(value) : value;
if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0) {
throw new Error(
`--${key.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`)} must be a nonnegative integer.`,
);
}
result[key] = count;
}
return result;
}
export function resolveBackupNamespace(namespace?: string): string {
const value = namespace ?? os.hostname().replace(/[^A-Za-z0-9._-]/g, "-");
if (!/^[A-Za-z0-9._-]{1,128}$/.test(value) || value === "." || value === "..") {
throw new Error(
"Backup namespace must be 1–128 letters, digits, dots, underscores or hyphens, and cannot be . or .. .",
);
}
return value;
}
export function createRemoteBackupKey(nowMs = Date.now()): string {
return `${new Date(nowMs)
.toISOString()
.replace(/[-:]/g, "")
.replace(/\.\d{3}Z$/, "Z")}-${randomBytes(4).toString("hex")}.tar.gz`;
}
/** Only consumer-owned keys with real UTC timestamps participate in listing or deletion. */
export function parseRemoteBackupTimestamp(key: string): number | undefined {
const match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z-[a-f0-9]{8}\.tar\.gz$/.exec(key);
if (!match) {
return undefined;
}
const iso = `${match[1]}-${match[2]}-${match[3]}T${match[4]}:${match[5]}:${match[6]}.000Z`;
const timestamp = Date.parse(iso);
return Number.isFinite(timestamp) && new Date(timestamp).toISOString() === iso
? timestamp
: undefined;
}
/** Keep the newest snapshot in each selected nonempty UTC calendar bucket (Monday weeks). */
export function selectBackupRetention(
keys: readonly string[],
options: BackupRetention,
): {
kept: string[];
deleted: string[];
} {
const backups = [...new Set(keys)]
.flatMap((key) => {
const timestamp = parseRemoteBackupTimestamp(key);
return timestamp === undefined ? [] : [{ key, timestamp }];
})
.toSorted((a, b) => b.timestamp - a.timestamp || b.key.localeCompare(a.key));
if (Object.values(options).every((value) => value === undefined)) {
return { kept: backups.map(({ key }) => key), deleted: [] };
}
const kept = new Set<string>(backups.slice(0, 1).map(({ key }) => key));
const buckets = {
keepDaily: new Set<string>(),
keepWeekly: new Set<string>(),
keepMonthly: new Set<string>(),
};
for (const { key, timestamp } of backups) {
const date = new Date(timestamp);
const day = date.toISOString().slice(0, 10);
const month = day.slice(0, 7);
date.setUTCDate(date.getUTCDate() - ((date.getUTCDay() + 6) % 7));
const week = date.toISOString().slice(0, 10);
for (const [period, bucket] of [
["keepDaily", day],
["keepWeekly", week],
["keepMonthly", month],
] as const) {
const seen = buckets[period];
if (!seen.has(bucket) && seen.size < (options[period] ?? 0)) {
kept.add(key);
seen.add(bucket);
}
}
}
return {
kept: backups.filter(({ key }) => kept.has(key)).map(({ key }) => key),
deleted: backups.filter(({ key }) => !kept.has(key)).map(({ key }) => key),
};
}

67
src/infra/backup-scratch.test.ts
View file

@ -429,41 +429,44 @@ it.each(["admitted", "creating"] as const)(
},
);
it("reclaims abandoned scratch while a live transaction protects its files", async () => {
const root = dirs.make("backup-scratch-lifetime-");
const abandoned = await createBackupScratchDirectory(root);
const active = await createBackupScratchDirectory(root);
const legacy = path.join(root, "openclaw-backup-Legacy");
const rollback = path.join(root, ".openclaw.package-backup-123-456");
for (const directory of [abandoned.directory, active.directory, legacy, rollback]) {
await fs.mkdir(directory, { recursive: true });
await fs.writeFile(path.join(directory, "config-0"), "synthetic backup input");
}
abandoned.release();
try {
const inspection = await maintainBackupScratch({ roots: [root], repair: false });
expect(inspection.reclaimed).toEqual([]);
expect(inspection.unchecked).toEqual(
expect.arrayContaining([active.directory, abandoned.directory]),
);
await expect(fs.readFile(path.join(abandoned.directory, "config-0"), "utf8")).resolves.toBe(
"synthetic backup input",
);
const repair = await maintainBackupScratch({ roots: [root], repair: true });
expect(repair.reclaimed).toEqual([abandoned.directory]);
expect(repair.active).toEqual([active.directory]);
expect(repair.warnings).toEqual([expect.stringContaining(legacy)]);
await expect(fs.stat(abandoned.directory)).rejects.toMatchObject({ code: "ENOENT" });
for (const directory of [active.directory, legacy, rollback]) {
await expect(fs.readFile(path.join(directory, "config-0"), "utf8")).resolves.toBe(
it.each(["config-0", "archive.tar.gz"])(
"reclaims abandoned %s scratch while a live transaction protects its files",
async (payloadName) => {
const root = dirs.make("backup-scratch-lifetime-");
const abandoned = await createBackupScratchDirectory(root);
const active = await createBackupScratchDirectory(root);
const legacy = path.join(root, "openclaw-backup-Legacy");
const rollback = path.join(root, ".openclaw.package-backup-123-456");
for (const directory of [abandoned.directory, active.directory, legacy, rollback]) {
await fs.mkdir(directory, { recursive: true });
await fs.writeFile(path.join(directory, payloadName), "synthetic backup input");
}
abandoned.release();
try {
const inspection = await maintainBackupScratch({ roots: [root], repair: false });
expect(inspection.reclaimed).toEqual([]);
expect(inspection.unchecked).toEqual(
expect.arrayContaining([active.directory, abandoned.directory]),
);
await expect(fs.readFile(path.join(abandoned.directory, payloadName), "utf8")).resolves.toBe(
"synthetic backup input",
);
const repair = await maintainBackupScratch({ roots: [root], repair: true });
expect(repair.reclaimed).toEqual([abandoned.directory]);
expect(repair.active).toEqual([active.directory]);
expect(repair.warnings).toEqual([expect.stringContaining(legacy)]);
await expect(fs.stat(abandoned.directory)).rejects.toMatchObject({ code: "ENOENT" });
for (const directory of [active.directory, legacy, rollback]) {
await expect(fs.readFile(path.join(directory, payloadName), "utf8")).resolves.toBe(
"synthetic backup input",
);
}
} finally {
await finishBackupScratch(active);
}
} finally {
await finishBackupScratch(active);
}
});
},
);
it.each(["payload", "directory"] as const)(
"retries retirement after failed %s cleanup",

Some files were not shown because too many files have changed in this diff Show more