diff --git a/backend/internal/cmds/export.go b/backend/internal/cmds/export.go index 1e12a837..b23c5fcd 100644 --- a/backend/internal/cmds/export.go +++ b/backend/internal/cmds/export.go @@ -2,7 +2,6 @@ package cmds import ( "context" - "errors" "fmt" "io" "os" @@ -36,12 +35,6 @@ func init() { // runExport orchestrates the export flow func runExport(ctx context.Context, flags exportFlags) error { - // The export includes the actor data, which a standalone Francis runtime keeps in its own store rather than in Pocket ID's database - // Exporting anyway would silently produce an archive missing that data, so refuse instead - if !common.EnvConfig.HasEmbeddedFrancisRuntime() { - return errors.New("exporting is not supported when FRANCIS_HOST points to a standalone Francis runtime: export Pocket ID's data and the runtime's data separately, using the runtime's own backup command for the latter") - } - db, pg, err := bootstrap.NewDatabase(ctx) if err != nil { return fmt.Errorf("failed to connect to database: %w", err) @@ -57,18 +50,27 @@ func runExport(ctx context.Context, flags exportFlags) error { _ = storage.Close() }() - // The actor host's data lives outside of the Pocket ID schema, so it's exported through Francis - providerOpts, err := bootstrap.ActorsProviderOptions(db, pg) - if err != nil { - return err + // The actor data lives outside of the Pocket ID schema, so it's exported through Francis + // A standalone runtime keeps it in its own store, out of reach from here, and the export service leaves the entry out of the archive when no provider is passed + var actorsProvider service.ActorsBackupProvider + if common.EnvConfig.HasEmbeddedFrancisRuntime() { + providerOpts, provErr := bootstrap.ActorsProviderOptions(db, pg) + if provErr != nil { + return provErr + } + + provider, provErr := bootstrap.NewActorsBackupProvider(ctx, providerOpts) + if provErr != nil { + return fmt.Errorf("failed to initialize the actor host's data provider: %w", provErr) + } + defer func() { + _ = provider.Close() + }() + + actorsProvider = provider + } else { + printRemoteActorDataNotice("The actor data is NOT included in this export", "back it up separately with: francis runtime backup -f actors.bin") } - actorsProvider, err := bootstrap.NewActorsBackupProvider(ctx, providerOpts) - if err != nil { - return fmt.Errorf("failed to initialize the actor host's data provider: %w", err) - } - defer func() { - _ = actorsProvider.Close() - }() exportService := service.NewExportService(db, storage, actorsProvider) diff --git a/backend/internal/cmds/francis_runtime.go b/backend/internal/cmds/francis_runtime.go new file mode 100644 index 00000000..226ca9aa --- /dev/null +++ b/backend/internal/cmds/francis_runtime.go @@ -0,0 +1,33 @@ +package cmds + +import ( + "archive/zip" + "fmt" + "os" + + "github.com/pocket-id/pocket-id/backend/internal/service" +) + +// printRemoteActorDataNotice warns that the command does not cover the actor data, which a standalone Francis runtime owns rather than Pocket ID +// It goes to stderr so it stays visible when the archive itself is streamed to stdout +func printRemoteActorDataNotice(consequence string, remedy string) { + fmt.Fprintf(os.Stderr, `WARNING: FRANCIS_HOST points to a standalone Francis runtime. + %s. + That data covers the app configuration, signup and one-time access tokens, device + login requests, LDAP sync state, and the schedule of the background jobs. + To cover it, %s. + +`, consequence, remedy) +} + +// ensureNoActorsBackup rejects an archive that carries the actor data when a standalone Francis runtime owns it +// Such an archive comes from a deployment with an embedded runtime, and restoring only its Pocket ID half would leave the runtime holding actor state belonging to a different deployment +func ensureNoActorsBackup(zipReader *zip.Reader) error { + for _, f := range zipReader.File { + if f.Name == service.ActorsBackupFileName { + return fmt.Errorf("this archive contains the actor data (%s) but FRANCIS_HOST points to a standalone Francis runtime, which owns that data instead: restore it into a deployment with an embedded runtime, or load the actor data into the runtime with 'francis runtime restore'", service.ActorsBackupFileName) + } + } + + return nil +} diff --git a/backend/internal/cmds/francis_runtime_test.go b/backend/internal/cmds/francis_runtime_test.go new file mode 100644 index 00000000..5999c1f2 --- /dev/null +++ b/backend/internal/cmds/francis_runtime_test.go @@ -0,0 +1,43 @@ +package cmds + +import ( + "archive/zip" + "bytes" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/pocket-id/pocket-id/backend/internal/service" +) + +func TestEnsureNoActorsBackup(t *testing.T) { + buildZip := func(t *testing.T, names ...string) *zip.Reader { + t.Helper() + + buf := &bytes.Buffer{} + zw := zip.NewWriter(buf) + for _, name := range names { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write([]byte("payload")) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + + zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + + return zr + } + + t.Run("accepts an archive without the actor data", func(t *testing.T) { + err := ensureNoActorsBackup(buildZip(t, "database.json", "uploads/logo.png")) + require.NoError(t, err) + }) + + t.Run("rejects an archive carrying the actor data", func(t *testing.T) { + err := ensureNoActorsBackup(buildZip(t, "database.json", service.ActorsBackupFileName)) + require.Error(t, err) + require.ErrorContains(t, err, service.ActorsBackupFileName) + }) +} diff --git a/backend/internal/cmds/import.go b/backend/internal/cmds/import.go index 73e89582..b10add59 100644 --- a/backend/internal/cmds/import.go +++ b/backend/internal/cmds/import.go @@ -47,10 +47,14 @@ func init() { // runImport handles the high-level orchestration of the import process func runImport(ctx context.Context, flags importFlags) error { - // The archive carries the actor data, which is restored into Pocket ID's own database and so is only reachable with the embedded runtime - // A standalone Francis runtime owns that data instead, and it has to be restored through the runtime itself - if !common.EnvConfig.HasEmbeddedFrancisRuntime() { - return errors.New("importing is not supported when FRANCIS_HOST points to a standalone Francis runtime: import Pocket ID's data and the runtime's data separately, using the runtime's own restore command for the latter") + // A standalone Francis runtime owns the actor data, so this import only covers what lives in Pocket ID's own database + // Nothing here can fence the replicas either, since they are hosts of the runtime's cluster rather than of a cluster in this database + embeddedRuntime := common.EnvConfig.HasEmbeddedFrancisRuntime() + if !embeddedRuntime { + printRemoteActorDataNotice( + "The actor data will NOT be restored, and Pocket ID replicas will NOT be stopped for you", + "stop every replica first, then restore the runtime with: francis runtime restore -f actors.bin", + ) } if !flags.Yes { @@ -81,35 +85,49 @@ func runImport(ctx context.Context, flags importFlags) error { } defer zipReader.Close() + // An archive carrying the actor data was taken from a deployment with an embedded runtime, and there is nowhere to put that data here + // Restoring only the Pocket ID half of it would leave the runtime holding actor state from a different deployment, so refuse rather than half-restore + if !embeddedRuntime { + err = ensureNoActorsBackup(&zipReader.Reader) + if err != nil { + return err + } + } + // Connect to the database without running migrations: the import re-creates the Pocket ID schema itself db, pg, err := bootstrap.ConnectDatabase(ctx) if err != nil { return err } - // The cluster admin talks to the same database as the actor host, so build its provider options the same way the host does - providerOpts, err := bootstrap.ActorsProviderOptions(db, pg) - if err != nil { - return err - } - - // Take exclusive access to the cluster so no Pocket ID replica is running while we overwrite the database - release, lost, err := acquireExclusiveAccess(ctx, providerOpts, flags.ForcefullyAcquireLock) - if err != nil { - return err - } - defer release() - - // Abort the import if exclusive access is lost partway through (for example if the lease can no longer be renewed) importCtx, cancel := context.WithCancel(ctx) defer cancel() - go func() { - select { - case <-lost: - cancel() - case <-importCtx.Done(): + + // Take exclusive access to the cluster so no Pocket ID replica is running while we overwrite the database + // The lease lives in the actor host's own tables, so it only exists when the runtime is embedded: with a standalone runtime the operator was told to stop the replicas instead + var providerOpts components.ProviderOptions + if embeddedRuntime { + // The cluster admin talks to the same database as the actor host, so build its provider options the same way the host does + providerOpts, err = bootstrap.ActorsProviderOptions(db, pg) + if err != nil { + return err } - }() + + release, lost, acquireErr := acquireExclusiveAccess(ctx, providerOpts, flags.ForcefullyAcquireLock) + if acquireErr != nil { + return acquireErr + } + defer release() + + // Abort the import if exclusive access is lost partway through (for example if the lease can no longer be renewed) + go func() { + select { + case <-lost: + cancel() + case <-importCtx.Done(): + } + }() + } // Init the storage provider storage, err := bootstrap.InitStorage(importCtx, db) @@ -122,15 +140,20 @@ func runImport(ctx context.Context, flags importFlags) error { _ = storage.Close() }() - // The actor host's data lives outside of the Pocket ID schema, so it's restored through Francis - // Restoring requires exclusive access to the cluster, which was acquired above - actorsProvider, err := bootstrap.NewActorsBackupProvider(importCtx, providerOpts) - if err != nil { - return fmt.Errorf("failed to initialize the actor host's data provider: %w", err) + // The actor data lives outside of the Pocket ID schema, so it's restored through Francis + // Restoring requires exclusive access to the cluster, which was acquired above, and the import service skips the actor data entirely when no provider is passed + var actorsProvider service.ActorsBackupProvider + if embeddedRuntime { + provider, provErr := bootstrap.NewActorsBackupProvider(importCtx, providerOpts) + if provErr != nil { + return fmt.Errorf("failed to initialize the actor host's data provider: %w", provErr) + } + defer func() { + _ = provider.Close() + }() + + actorsProvider = provider } - defer func() { - _ = actorsProvider.Close() - }() // Create the import service importService := service.NewImportService(db, storage, actorsProvider) diff --git a/backend/internal/service/actors_backup.go b/backend/internal/service/actors_backup.go index 53fb4eca..3fe73847 100644 --- a/backend/internal/service/actors_backup.go +++ b/backend/internal/service/actors_backup.go @@ -5,9 +5,9 @@ import ( "io" ) -// actorsBackupFileName is the name of the entry in the export ZIP that contains the actor host's data +// ActorsBackupFileName is the name of the entry in the export ZIP that contains the actor host's data // The payload is Francis' own backup stream, which is a binary, versioned, provider-neutral format -const actorsBackupFileName = "francis.bin" +const ActorsBackupFileName = "francis.bin" // ActorsBackupProvider backs up and restores the actor host's data: actor state, alarms, and dead-lettered jobs. type ActorsBackupProvider interface { diff --git a/backend/internal/service/actors_backup_test.go b/backend/internal/service/actors_backup_test.go index c57f4943..1e1603e3 100644 --- a/backend/internal/service/actors_backup_test.go +++ b/backend/internal/service/actors_backup_test.go @@ -45,7 +45,7 @@ func TestExportActorsBackup(t *testing.T) { files := writeActorsBackupZip(t, NewExportService(nil, nil, actors)) - require.Equal(t, map[string][]byte{actorsBackupFileName: actors.backupData}, files) + require.Equal(t, map[string][]byte{ActorsBackupFileName: actors.backupData}, files) }) t.Run("adds nothing without a provider", func(t *testing.T) { @@ -69,7 +69,7 @@ func TestImportActorsBackup(t *testing.T) { actors := &stubActorsBackupProvider{} files := readZip(t, buildZip(t, map[string][]byte{ "database.json": []byte("{}"), - actorsBackupFileName: []byte("francis-backup-payload"), + ActorsBackupFileName: []byte("francis-backup-payload"), })) err := NewImportService(nil, nil, actors).importActorsBackup(t.Context(), files) @@ -92,7 +92,7 @@ func TestImportActorsBackup(t *testing.T) { t.Run("surfaces restore errors", func(t *testing.T) { actors := &stubActorsBackupProvider{restoreErr: errors.New("a host is still connected")} - files := readZip(t, buildZip(t, map[string][]byte{actorsBackupFileName: []byte("francis-backup-payload")})) + files := readZip(t, buildZip(t, map[string][]byte{ActorsBackupFileName: []byte("francis-backup-payload")})) err := NewImportService(nil, nil, actors).importActorsBackup(t.Context(), files) diff --git a/backend/internal/service/export_service.go b/backend/internal/service/export_service.go index 9ba4793c..b612f4c2 100644 --- a/backend/internal/service/export_service.go +++ b/backend/internal/service/export_service.go @@ -248,9 +248,9 @@ func (s *ExportService) addActorsBackupToZip(ctx context.Context, zipWriter *zip return nil } - w, err := zipWriter.Create(actorsBackupFileName) + w, err := zipWriter.Create(ActorsBackupFileName) if err != nil { - return fmt.Errorf("failed to create %s in zip: %w", actorsBackupFileName, err) + return fmt.Errorf("failed to create %s in zip: %w", ActorsBackupFileName, err) } err = s.actors.Backup(ctx, w) diff --git a/backend/internal/service/import_service.go b/backend/internal/service/import_service.go index 65f9597d..1946026c 100644 --- a/backend/internal/service/import_service.go +++ b/backend/internal/service/import_service.go @@ -81,20 +81,20 @@ func (s *ImportService) importActorsBackup(ctx context.Context, files []*zip.Fil var backupFile *zip.File for _, f := range files { - if f.Name == actorsBackupFileName { + if f.Name == ActorsBackupFileName { backupFile = f break } } if backupFile == nil { // Archives exported before Pocket ID included the actor host's data don't have that entry, in which case the existing data is left untouched - slog.WarnContext(ctx, "The archive does not contain the actor host's data, which will be left unchanged", slog.String("file", actorsBackupFileName)) + slog.WarnContext(ctx, "The archive does not contain the actor host's data, which will be left unchanged", slog.String("file", ActorsBackupFileName)) return nil } rc, err := backupFile.Open() if err != nil { - return fmt.Errorf("failed to open %s: %w", actorsBackupFileName, err) + return fmt.Errorf("failed to open %s: %w", ActorsBackupFileName, err) } defer rc.Close()