feat: export and import Pocket ID's own data against a standalone runtime

With a standalone Francis runtime the actor data lives in the runtime's
store, which Pocket ID cannot reach: the Francis protocol has no backup
or restore operation, and a restore refuses to run while any host is
connected, so a CLI that joined the cluster would block its own restore.

Rather than refusing outright, both commands now cover everything Pocket
ID does own and say plainly what they leave out, pointing at the
runtime's own backup and restore commands for the rest. The export writes
no francis.bin entry, which the import side already tolerates, and the
import refuses an archive that carries one, since restoring only its
Pocket ID half would leave the runtime holding another deployment's actor
state. The import also skips the exclusive-access lease, which lives in
the actor tables of a database this deployment does not use, and warns
that the replicas have to be stopped by hand.

backend/internal/cmds/export.go

@@ -2,7 +2,6 @@ package cmds
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -36,12 +35,6 @@ func init() {
 
 // runExport orchestrates the export flow
 func runExport(ctx context.Context, flags exportFlags) error {
-	// The export includes the actor data, which a standalone Francis runtime keeps in its own store rather than in Pocket ID's database
-	// Exporting anyway would silently produce an archive missing that data, so refuse instead
-	if !common.EnvConfig.HasEmbeddedFrancisRuntime() {
-		return errors.New("exporting is not supported when FRANCIS_HOST points to a standalone Francis runtime: export Pocket ID's data and the runtime's data separately, using the runtime's own backup command for the latter")
-	}
-
 	db, pg, err := bootstrap.NewDatabase(ctx)
 	if err != nil {
 		return fmt.Errorf("failed to connect to database: %w", err)
@@ -57,18 +50,27 @@ func runExport(ctx context.Context, flags exportFlags) error {
 		_ = storage.Close()
 	}()
 
-	// The actor host's data lives outside of the Pocket ID schema, so it's exported through Francis
-	providerOpts, err := bootstrap.ActorsProviderOptions(db, pg)
-	if err != nil {
-		return err
+	// The actor data lives outside of the Pocket ID schema, so it's exported through Francis
+	// A standalone runtime keeps it in its own store, out of reach from here, and the export service leaves the entry out of the archive when no provider is passed
+	var actorsProvider service.ActorsBackupProvider
+	if common.EnvConfig.HasEmbeddedFrancisRuntime() {
+		providerOpts, provErr := bootstrap.ActorsProviderOptions(db, pg)
+		if provErr != nil {
+			return provErr
+		}
+
+		provider, provErr := bootstrap.NewActorsBackupProvider(ctx, providerOpts)
+		if provErr != nil {
+			return fmt.Errorf("failed to initialize the actor host's data provider: %w", provErr)
+		}
+		defer func() {
+			_ = provider.Close()
+		}()
+
+		actorsProvider = provider
+	} else {
+		printRemoteActorDataNotice("The actor data is NOT included in this export", "back it up separately with: francis runtime backup -f actors.bin")
 	}
-	actorsProvider, err := bootstrap.NewActorsBackupProvider(ctx, providerOpts)
-	if err != nil {
-		return fmt.Errorf("failed to initialize the actor host's data provider: %w", err)
-	}
-	defer func() {
-		_ = actorsProvider.Close()
-	}()
 
 	exportService := service.NewExportService(db, storage, actorsProvider)
 

backend/internal/cmds/francis_runtime.go

@@ -0,0 +1,33 @@
+package cmds
+
+import (
+	"archive/zip"
+	"fmt"
+	"os"
+
+	"github.com/pocket-id/pocket-id/backend/internal/service"
+)
+
+// printRemoteActorDataNotice warns that the command does not cover the actor data, which a standalone Francis runtime owns rather than Pocket ID
+// It goes to stderr so it stays visible when the archive itself is streamed to stdout
+func printRemoteActorDataNotice(consequence string, remedy string) {
+	fmt.Fprintf(os.Stderr, `WARNING: FRANCIS_HOST points to a standalone Francis runtime.
+  %s.
+  That data covers the app configuration, signup and one-time access tokens, device
+  login requests, LDAP sync state, and the schedule of the background jobs.
+  To cover it, %s.
+
+`, consequence, remedy)
+}
+
+// ensureNoActorsBackup rejects an archive that carries the actor data when a standalone Francis runtime owns it
+// Such an archive comes from a deployment with an embedded runtime, and restoring only its Pocket ID half would leave the runtime holding actor state belonging to a different deployment
+func ensureNoActorsBackup(zipReader *zip.Reader) error {
+	for _, f := range zipReader.File {
+		if f.Name == service.ActorsBackupFileName {
+			return fmt.Errorf("this archive contains the actor data (%s) but FRANCIS_HOST points to a standalone Francis runtime, which owns that data instead: restore it into a deployment with an embedded runtime, or load the actor data into the runtime with 'francis runtime restore'", service.ActorsBackupFileName)
+		}
+	}
+
+	return nil
+}

backend/internal/cmds/francis_runtime_test.go

@@ -0,0 +1,43 @@
+package cmds
+
+import (
+	"archive/zip"
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/pocket-id/pocket-id/backend/internal/service"
+)
+
+func TestEnsureNoActorsBackup(t *testing.T) {
+	buildZip := func(t *testing.T, names ...string) *zip.Reader {
+		t.Helper()
+
+		buf := &bytes.Buffer{}
+		zw := zip.NewWriter(buf)
+		for _, name := range names {
+			w, err := zw.Create(name)
+			require.NoError(t, err)
+			_, err = w.Write([]byte("payload"))
+			require.NoError(t, err)
+		}
+		require.NoError(t, zw.Close())
+
+		zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
+		require.NoError(t, err)
+
+		return zr
+	}
+
+	t.Run("accepts an archive without the actor data", func(t *testing.T) {
+		err := ensureNoActorsBackup(buildZip(t, "database.json", "uploads/logo.png"))
+		require.NoError(t, err)
+	})
+
+	t.Run("rejects an archive carrying the actor data", func(t *testing.T) {
+		err := ensureNoActorsBackup(buildZip(t, "database.json", service.ActorsBackupFileName))
+		require.Error(t, err)
+		require.ErrorContains(t, err, service.ActorsBackupFileName)
+	})
+}

backend/internal/cmds/import.go

@@ -47,10 +47,14 @@ func init() {
 
 // runImport handles the high-level orchestration of the import process
 func runImport(ctx context.Context, flags importFlags) error {
-	// The archive carries the actor data, which is restored into Pocket ID's own database and so is only reachable with the embedded runtime
-	// A standalone Francis runtime owns that data instead, and it has to be restored through the runtime itself
-	if !common.EnvConfig.HasEmbeddedFrancisRuntime() {
-		return errors.New("importing is not supported when FRANCIS_HOST points to a standalone Francis runtime: import Pocket ID's data and the runtime's data separately, using the runtime's own restore command for the latter")
+	// A standalone Francis runtime owns the actor data, so this import only covers what lives in Pocket ID's own database
+	// Nothing here can fence the replicas either, since they are hosts of the runtime's cluster rather than of a cluster in this database
+	embeddedRuntime := common.EnvConfig.HasEmbeddedFrancisRuntime()
+	if !embeddedRuntime {
+		printRemoteActorDataNotice(
+			"The actor data will NOT be restored, and Pocket ID replicas will NOT be stopped for you",
+			"stop every replica first, then restore the runtime with: francis runtime restore -f actors.bin",
+		)
 	}
 
 	if !flags.Yes {
@@ -81,35 +85,49 @@ func runImport(ctx context.Context, flags importFlags) error {
 	}
 	defer zipReader.Close()
 
+	// An archive carrying the actor data was taken from a deployment with an embedded runtime, and there is nowhere to put that data here
+	// Restoring only the Pocket ID half of it would leave the runtime holding actor state from a different deployment, so refuse rather than half-restore
+	if !embeddedRuntime {
+		err = ensureNoActorsBackup(&zipReader.Reader)
+		if err != nil {
+			return err
+		}
+	}
+
 	// Connect to the database without running migrations: the import re-creates the Pocket ID schema itself
 	db, pg, err := bootstrap.ConnectDatabase(ctx)
 	if err != nil {
 		return err
 	}
 
-	// The cluster admin talks to the same database as the actor host, so build its provider options the same way the host does
-	providerOpts, err := bootstrap.ActorsProviderOptions(db, pg)
-	if err != nil {
-		return err
-	}
-
-	// Take exclusive access to the cluster so no Pocket ID replica is running while we overwrite the database
-	release, lost, err := acquireExclusiveAccess(ctx, providerOpts, flags.ForcefullyAcquireLock)
-	if err != nil {
-		return err
-	}
-	defer release()
-
-	// Abort the import if exclusive access is lost partway through (for example if the lease can no longer be renewed)
 	importCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	go func() {
-		select {
-		case <-lost:
-			cancel()
-		case <-importCtx.Done():
+
+	// Take exclusive access to the cluster so no Pocket ID replica is running while we overwrite the database
+	// The lease lives in the actor host's own tables, so it only exists when the runtime is embedded: with a standalone runtime the operator was told to stop the replicas instead
+	var providerOpts components.ProviderOptions
+	if embeddedRuntime {
+		// The cluster admin talks to the same database as the actor host, so build its provider options the same way the host does
+		providerOpts, err = bootstrap.ActorsProviderOptions(db, pg)
+		if err != nil {
+			return err
 		}
-	}()
+
+		release, lost, acquireErr := acquireExclusiveAccess(ctx, providerOpts, flags.ForcefullyAcquireLock)
+		if acquireErr != nil {
+			return acquireErr
+		}
+		defer release()
+
+		// Abort the import if exclusive access is lost partway through (for example if the lease can no longer be renewed)
+		go func() {
+			select {
+			case <-lost:
+				cancel()
+			case <-importCtx.Done():
+			}
+		}()
+	}
 
 	// Init the storage provider
 	storage, err := bootstrap.InitStorage(importCtx, db)
@@ -122,15 +140,20 @@ func runImport(ctx context.Context, flags importFlags) error {
 		_ = storage.Close()
 	}()
 
-	// The actor host's data lives outside of the Pocket ID schema, so it's restored through Francis
-	// Restoring requires exclusive access to the cluster, which was acquired above
-	actorsProvider, err := bootstrap.NewActorsBackupProvider(importCtx, providerOpts)
-	if err != nil {
-		return fmt.Errorf("failed to initialize the actor host's data provider: %w", err)
+	// The actor data lives outside of the Pocket ID schema, so it's restored through Francis
+	// Restoring requires exclusive access to the cluster, which was acquired above, and the import service skips the actor data entirely when no provider is passed
+	var actorsProvider service.ActorsBackupProvider
+	if embeddedRuntime {
+		provider, provErr := bootstrap.NewActorsBackupProvider(importCtx, providerOpts)
+		if provErr != nil {
+			return fmt.Errorf("failed to initialize the actor host's data provider: %w", provErr)
+		}
+		defer func() {
+			_ = provider.Close()
+		}()
+
+		actorsProvider = provider
 	}
-	defer func() {
-		_ = actorsProvider.Close()
-	}()
 
 	// Create the import service
 	importService := service.NewImportService(db, storage, actorsProvider)

backend/internal/service/actors_backup.go

@@ -5,9 +5,9 @@ import (
 	"io"
 )
 
-// actorsBackupFileName is the name of the entry in the export ZIP that contains the actor host's data
+// ActorsBackupFileName is the name of the entry in the export ZIP that contains the actor host's data
 // The payload is Francis' own backup stream, which is a binary, versioned, provider-neutral format
-const actorsBackupFileName = "francis.bin"
+const ActorsBackupFileName = "francis.bin"
 
 // ActorsBackupProvider backs up and restores the actor host's data: actor state, alarms, and dead-lettered jobs.
 type ActorsBackupProvider interface {

backend/internal/service/actors_backup_test.go

@@ -45,7 +45,7 @@ func TestExportActorsBackup(t *testing.T) {
 
 		files := writeActorsBackupZip(t, NewExportService(nil, nil, actors))
 
-		require.Equal(t, map[string][]byte{actorsBackupFileName: actors.backupData}, files)
+		require.Equal(t, map[string][]byte{ActorsBackupFileName: actors.backupData}, files)
 	})
 
 	t.Run("adds nothing without a provider", func(t *testing.T) {
@@ -69,7 +69,7 @@ func TestImportActorsBackup(t *testing.T) {
 		actors := &stubActorsBackupProvider{}
 		files := readZip(t, buildZip(t, map[string][]byte{
 			"database.json":      []byte("{}"),
-			actorsBackupFileName: []byte("francis-backup-payload"),
+			ActorsBackupFileName: []byte("francis-backup-payload"),
 		}))
 
 		err := NewImportService(nil, nil, actors).importActorsBackup(t.Context(), files)
@@ -92,7 +92,7 @@ func TestImportActorsBackup(t *testing.T) {
 
 	t.Run("surfaces restore errors", func(t *testing.T) {
 		actors := &stubActorsBackupProvider{restoreErr: errors.New("a host is still connected")}
-		files := readZip(t, buildZip(t, map[string][]byte{actorsBackupFileName: []byte("francis-backup-payload")}))
+		files := readZip(t, buildZip(t, map[string][]byte{ActorsBackupFileName: []byte("francis-backup-payload")}))
 
 		err := NewImportService(nil, nil, actors).importActorsBackup(t.Context(), files)
 

backend/internal/service/export_service.go

@@ -248,9 +248,9 @@ func (s *ExportService) addActorsBackupToZip(ctx context.Context, zipWriter *zip
 		return nil
 	}
 
-	w, err := zipWriter.Create(actorsBackupFileName)
+	w, err := zipWriter.Create(ActorsBackupFileName)
 	if err != nil {
-		return fmt.Errorf("failed to create %s in zip: %w", actorsBackupFileName, err)
+		return fmt.Errorf("failed to create %s in zip: %w", ActorsBackupFileName, err)
 	}
 
 	err = s.actors.Backup(ctx, w)

backend/internal/service/import_service.go

@@ -81,20 +81,20 @@ func (s *ImportService) importActorsBackup(ctx context.Context, files []*zip.Fil
 
 	var backupFile *zip.File
 	for _, f := range files {
-		if f.Name == actorsBackupFileName {
+		if f.Name == ActorsBackupFileName {
 			backupFile = f
 			break
 		}
 	}
 	if backupFile == nil {
 		// Archives exported before Pocket ID included the actor host's data don't have that entry, in which case the existing data is left untouched
-		slog.WarnContext(ctx, "The archive does not contain the actor host's data, which will be left unchanged", slog.String("file", actorsBackupFileName))
+		slog.WarnContext(ctx, "The archive does not contain the actor host's data, which will be left unchanged", slog.String("file", ActorsBackupFileName))
 		return nil
 	}
 
 	rc, err := backupFile.Open()
 	if err != nil {
-		return fmt.Errorf("failed to open %s: %w", actorsBackupFileName, err)
+		return fmt.Errorf("failed to open %s: %w", ActorsBackupFileName, err)
 	}
 	defer rc.Close()