From 5cb420a774f4215d9d06e6fa2feae8315933e523 Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:22:40 +0000 Subject: [PATCH] feat: export and import Pocket ID's own data against a standalone runtime With a standalone Francis runtime the actor data lives in the runtime's store, which Pocket ID cannot reach: the Francis protocol has no backup or restore operation, and a restore refuses to run while any host is connected, so a CLI that joined the cluster would block its own restore. Rather than refusing outright, both commands now cover everything Pocket ID does own and say plainly what they leave out, pointing at the runtime's own backup and restore commands for the rest. The export writes no francis.bin entry, which the import side already tolerates, and the import refuses an archive that carries one, since restoring only its Pocket ID half would leave the runtime holding another deployment's actor state. The import also skips the exclusive-access lease, which lives in the actor tables of a database this deployment does not use, and warns that the replicas have to be stopped by hand. --- backend/internal/cmds/export.go | 38 ++++---- backend/internal/cmds/francis_runtime.go | 33 +++++++ backend/internal/cmds/francis_runtime_test.go | 43 +++++++++ backend/internal/cmds/import.go | 87 ++++++++++++------- backend/internal/service/actors_backup.go | 4 +- .../internal/service/actors_backup_test.go | 6 +- backend/internal/service/export_service.go | 4 +- backend/internal/service/import_service.go | 6 +- 8 files changed, 161 insertions(+), 60 deletions(-) create mode 100644 backend/internal/cmds/francis_runtime.go create mode 100644 backend/internal/cmds/francis_runtime_test.go diff --git a/backend/internal/cmds/export.go b/backend/internal/cmds/export.go index 1e12a837..b23c5fcd 100644 --- a/backend/internal/cmds/export.go +++ b/backend/internal/cmds/export.go @@ -2,7 +2,6 @@ package cmds import ( "context" - "errors" "fmt" "io" "os" @@ -36,12 +35,6 @@ func init() { // runExport orchestrates the export flow func runExport(ctx context.Context, flags exportFlags) error { - // The export includes the actor data, which a standalone Francis runtime keeps in its own store rather than in Pocket ID's database - // Exporting anyway would silently produce an archive missing that data, so refuse instead - if !common.EnvConfig.HasEmbeddedFrancisRuntime() { - return errors.New("exporting is not supported when FRANCIS_HOST points to a standalone Francis runtime: export Pocket ID's data and the runtime's data separately, using the runtime's own backup command for the latter") - } - db, pg, err := bootstrap.NewDatabase(ctx) if err != nil { return fmt.Errorf("failed to connect to database: %w", err) @@ -57,18 +50,27 @@ func runExport(ctx context.Context, flags exportFlags) error { _ = storage.Close() }() - // The actor host's data lives outside of the Pocket ID schema, so it's exported through Francis - providerOpts, err := bootstrap.ActorsProviderOptions(db, pg) - if err != nil { - return err + // The actor data lives outside of the Pocket ID schema, so it's exported through Francis + // A standalone runtime keeps it in its own store, out of reach from here, and the export service leaves the entry out of the archive when no provider is passed + var actorsProvider service.ActorsBackupProvider + if common.EnvConfig.HasEmbeddedFrancisRuntime() { + providerOpts, provErr := bootstrap.ActorsProviderOptions(db, pg) + if provErr != nil { + return provErr + } + + provider, provErr := bootstrap.NewActorsBackupProvider(ctx, providerOpts) + if provErr != nil { + return fmt.Errorf("failed to initialize the actor host's data provider: %w", provErr) + } + defer func() { + _ = provider.Close() + }() + + actorsProvider = provider + } else { + printRemoteActorDataNotice("The actor data is NOT included in this export", "back it up separately with: francis runtime backup -f actors.bin") } - actorsProvider, err := bootstrap.NewActorsBackupProvider(ctx, providerOpts) - if err != nil { - return fmt.Errorf("failed to initialize the actor host's data provider: %w", err) - } - defer func() { - _ = actorsProvider.Close() - }() exportService := service.NewExportService(db, storage, actorsProvider) diff --git a/backend/internal/cmds/francis_runtime.go b/backend/internal/cmds/francis_runtime.go new file mode 100644 index 00000000..226ca9aa --- /dev/null +++ b/backend/internal/cmds/francis_runtime.go @@ -0,0 +1,33 @@ +package cmds + +import ( + "archive/zip" + "fmt" + "os" + + "github.com/pocket-id/pocket-id/backend/internal/service" +) + +// printRemoteActorDataNotice warns that the command does not cover the actor data, which a standalone Francis runtime owns rather than Pocket ID +// It goes to stderr so it stays visible when the archive itself is streamed to stdout +func printRemoteActorDataNotice(consequence string, remedy string) { + fmt.Fprintf(os.Stderr, `WARNING: FRANCIS_HOST points to a standalone Francis runtime. + %s. + That data covers the app configuration, signup and one-time access tokens, device + login requests, LDAP sync state, and the schedule of the background jobs. + To cover it, %s. + +`, consequence, remedy) +} + +// ensureNoActorsBackup rejects an archive that carries the actor data when a standalone Francis runtime owns it +// Such an archive comes from a deployment with an embedded runtime, and restoring only its Pocket ID half would leave the runtime holding actor state belonging to a different deployment +func ensureNoActorsBackup(zipReader *zip.Reader) error { + for _, f := range zipReader.File { + if f.Name == service.ActorsBackupFileName { + return fmt.Errorf("this archive contains the actor data (%s) but FRANCIS_HOST points to a standalone Francis runtime, which owns that data instead: restore it into a deployment with an embedded runtime, or load the actor data into the runtime with 'francis runtime restore'", service.ActorsBackupFileName) + } + } + + return nil +} diff --git a/backend/internal/cmds/francis_runtime_test.go b/backend/internal/cmds/francis_runtime_test.go new file mode 100644 index 00000000..5999c1f2 --- /dev/null +++ b/backend/internal/cmds/francis_runtime_test.go @@ -0,0 +1,43 @@ +package cmds + +import ( + "archive/zip" + "bytes" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/pocket-id/pocket-id/backend/internal/service" +) + +func TestEnsureNoActorsBackup(t *testing.T) { + buildZip := func(t *testing.T, names ...string) *zip.Reader { + t.Helper() + + buf := &bytes.Buffer{} + zw := zip.NewWriter(buf) + for _, name := range names { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write([]byte("payload")) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + + zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + + return zr + } + + t.Run("accepts an archive without the actor data", func(t *testing.T) { + err := ensureNoActorsBackup(buildZip(t, "database.json", "uploads/logo.png")) + require.NoError(t, err) + }) + + t.Run("rejects an archive carrying the actor data", func(t *testing.T) { + err := ensureNoActorsBackup(buildZip(t, "database.json", service.ActorsBackupFileName)) + require.Error(t, err) + require.ErrorContains(t, err, service.ActorsBackupFileName) + }) +} diff --git a/backend/internal/cmds/import.go b/backend/internal/cmds/import.go index 73e89582..b10add59 100644 --- a/backend/internal/cmds/import.go +++ b/backend/internal/cmds/import.go @@ -47,10 +47,14 @@ func init() { // runImport handles the high-level orchestration of the import process func runImport(ctx context.Context, flags importFlags) error { - // The archive carries the actor data, which is restored into Pocket ID's own database and so is only reachable with the embedded runtime - // A standalone Francis runtime owns that data instead, and it has to be restored through the runtime itself - if !common.EnvConfig.HasEmbeddedFrancisRuntime() { - return errors.New("importing is not supported when FRANCIS_HOST points to a standalone Francis runtime: import Pocket ID's data and the runtime's data separately, using the runtime's own restore command for the latter") + // A standalone Francis runtime owns the actor data, so this import only covers what lives in Pocket ID's own database + // Nothing here can fence the replicas either, since they are hosts of the runtime's cluster rather than of a cluster in this database + embeddedRuntime := common.EnvConfig.HasEmbeddedFrancisRuntime() + if !embeddedRuntime { + printRemoteActorDataNotice( + "The actor data will NOT be restored, and Pocket ID replicas will NOT be stopped for you", + "stop every replica first, then restore the runtime with: francis runtime restore -f actors.bin", + ) } if !flags.Yes { @@ -81,35 +85,49 @@ func runImport(ctx context.Context, flags importFlags) error { } defer zipReader.Close() + // An archive carrying the actor data was taken from a deployment with an embedded runtime, and there is nowhere to put that data here + // Restoring only the Pocket ID half of it would leave the runtime holding actor state from a different deployment, so refuse rather than half-restore + if !embeddedRuntime { + err = ensureNoActorsBackup(&zipReader.Reader) + if err != nil { + return err + } + } + // Connect to the database without running migrations: the import re-creates the Pocket ID schema itself db, pg, err := bootstrap.ConnectDatabase(ctx) if err != nil { return err } - // The cluster admin talks to the same database as the actor host, so build its provider options the same way the host does - providerOpts, err := bootstrap.ActorsProviderOptions(db, pg) - if err != nil { - return err - } - - // Take exclusive access to the cluster so no Pocket ID replica is running while we overwrite the database - release, lost, err := acquireExclusiveAccess(ctx, providerOpts, flags.ForcefullyAcquireLock) - if err != nil { - return err - } - defer release() - - // Abort the import if exclusive access is lost partway through (for example if the lease can no longer be renewed) importCtx, cancel := context.WithCancel(ctx) defer cancel() - go func() { - select { - case <-lost: - cancel() - case <-importCtx.Done(): + + // Take exclusive access to the cluster so no Pocket ID replica is running while we overwrite the database + // The lease lives in the actor host's own tables, so it only exists when the runtime is embedded: with a standalone runtime the operator was told to stop the replicas instead + var providerOpts components.ProviderOptions + if embeddedRuntime { + // The cluster admin talks to the same database as the actor host, so build its provider options the same way the host does + providerOpts, err = bootstrap.ActorsProviderOptions(db, pg) + if err != nil { + return err } - }() + + release, lost, acquireErr := acquireExclusiveAccess(ctx, providerOpts, flags.ForcefullyAcquireLock) + if acquireErr != nil { + return acquireErr + } + defer release() + + // Abort the import if exclusive access is lost partway through (for example if the lease can no longer be renewed) + go func() { + select { + case <-lost: + cancel() + case <-importCtx.Done(): + } + }() + } // Init the storage provider storage, err := bootstrap.InitStorage(importCtx, db) @@ -122,15 +140,20 @@ func runImport(ctx context.Context, flags importFlags) error { _ = storage.Close() }() - // The actor host's data lives outside of the Pocket ID schema, so it's restored through Francis - // Restoring requires exclusive access to the cluster, which was acquired above - actorsProvider, err := bootstrap.NewActorsBackupProvider(importCtx, providerOpts) - if err != nil { - return fmt.Errorf("failed to initialize the actor host's data provider: %w", err) + // The actor data lives outside of the Pocket ID schema, so it's restored through Francis + // Restoring requires exclusive access to the cluster, which was acquired above, and the import service skips the actor data entirely when no provider is passed + var actorsProvider service.ActorsBackupProvider + if embeddedRuntime { + provider, provErr := bootstrap.NewActorsBackupProvider(importCtx, providerOpts) + if provErr != nil { + return fmt.Errorf("failed to initialize the actor host's data provider: %w", provErr) + } + defer func() { + _ = provider.Close() + }() + + actorsProvider = provider } - defer func() { - _ = actorsProvider.Close() - }() // Create the import service importService := service.NewImportService(db, storage, actorsProvider) diff --git a/backend/internal/service/actors_backup.go b/backend/internal/service/actors_backup.go index 53fb4eca..3fe73847 100644 --- a/backend/internal/service/actors_backup.go +++ b/backend/internal/service/actors_backup.go @@ -5,9 +5,9 @@ import ( "io" ) -// actorsBackupFileName is the name of the entry in the export ZIP that contains the actor host's data +// ActorsBackupFileName is the name of the entry in the export ZIP that contains the actor host's data // The payload is Francis' own backup stream, which is a binary, versioned, provider-neutral format -const actorsBackupFileName = "francis.bin" +const ActorsBackupFileName = "francis.bin" // ActorsBackupProvider backs up and restores the actor host's data: actor state, alarms, and dead-lettered jobs. type ActorsBackupProvider interface { diff --git a/backend/internal/service/actors_backup_test.go b/backend/internal/service/actors_backup_test.go index c57f4943..1e1603e3 100644 --- a/backend/internal/service/actors_backup_test.go +++ b/backend/internal/service/actors_backup_test.go @@ -45,7 +45,7 @@ func TestExportActorsBackup(t *testing.T) { files := writeActorsBackupZip(t, NewExportService(nil, nil, actors)) - require.Equal(t, map[string][]byte{actorsBackupFileName: actors.backupData}, files) + require.Equal(t, map[string][]byte{ActorsBackupFileName: actors.backupData}, files) }) t.Run("adds nothing without a provider", func(t *testing.T) { @@ -69,7 +69,7 @@ func TestImportActorsBackup(t *testing.T) { actors := &stubActorsBackupProvider{} files := readZip(t, buildZip(t, map[string][]byte{ "database.json": []byte("{}"), - actorsBackupFileName: []byte("francis-backup-payload"), + ActorsBackupFileName: []byte("francis-backup-payload"), })) err := NewImportService(nil, nil, actors).importActorsBackup(t.Context(), files) @@ -92,7 +92,7 @@ func TestImportActorsBackup(t *testing.T) { t.Run("surfaces restore errors", func(t *testing.T) { actors := &stubActorsBackupProvider{restoreErr: errors.New("a host is still connected")} - files := readZip(t, buildZip(t, map[string][]byte{actorsBackupFileName: []byte("francis-backup-payload")})) + files := readZip(t, buildZip(t, map[string][]byte{ActorsBackupFileName: []byte("francis-backup-payload")})) err := NewImportService(nil, nil, actors).importActorsBackup(t.Context(), files) diff --git a/backend/internal/service/export_service.go b/backend/internal/service/export_service.go index 9ba4793c..b612f4c2 100644 --- a/backend/internal/service/export_service.go +++ b/backend/internal/service/export_service.go @@ -248,9 +248,9 @@ func (s *ExportService) addActorsBackupToZip(ctx context.Context, zipWriter *zip return nil } - w, err := zipWriter.Create(actorsBackupFileName) + w, err := zipWriter.Create(ActorsBackupFileName) if err != nil { - return fmt.Errorf("failed to create %s in zip: %w", actorsBackupFileName, err) + return fmt.Errorf("failed to create %s in zip: %w", ActorsBackupFileName, err) } err = s.actors.Backup(ctx, w) diff --git a/backend/internal/service/import_service.go b/backend/internal/service/import_service.go index 65f9597d..1946026c 100644 --- a/backend/internal/service/import_service.go +++ b/backend/internal/service/import_service.go @@ -81,20 +81,20 @@ func (s *ImportService) importActorsBackup(ctx context.Context, files []*zip.Fil var backupFile *zip.File for _, f := range files { - if f.Name == actorsBackupFileName { + if f.Name == ActorsBackupFileName { backupFile = f break } } if backupFile == nil { // Archives exported before Pocket ID included the actor host's data don't have that entry, in which case the existing data is left untouched - slog.WarnContext(ctx, "The archive does not contain the actor host's data, which will be left unchanged", slog.String("file", actorsBackupFileName)) + slog.WarnContext(ctx, "The archive does not contain the actor host's data, which will be left unchanged", slog.String("file", ActorsBackupFileName)) return nil } rc, err := backupFile.Open() if err != nil { - return fmt.Errorf("failed to open %s: %w", actorsBackupFileName, err) + return fmt.Errorf("failed to open %s: %w", ActorsBackupFileName, err) } defer rc.Close()