From d296b59a9eabf6813b3f146dfee3161e789a2a41 Mon Sep 17 00:00:00 2001 From: Tobias Gesellchen Date: Fri, 6 Mar 2026 21:50:41 +0100 Subject: [PATCH] Add/update docs. Some are only in preparation for future improvements and features (#99) --- .github/markdown-link-check.json | 3 + .github/workflows/ci.yml | 23 + docs/README.md | 98 +- docs/SCMUDC-ENRICHMENT-IMPLEMENTATION.md | 190 ++++ docs/SUMMARY.md | 10 + docs/analysis/IOT-CONFIG-SUMMARY.md | 189 ++++ docs/analysis/IOT-CONFIGURATION-ANALYSIS.md | 370 +++++++ docs/analysis/UPSTREAM-URLS.md | 14 +- docs/analysis/WIKI-COMPARISON.md | 142 +-- docs/concepts/README.md | 181 ++++ docs/concepts/implementation-plan.md | 355 +++++++ docs/concepts/implementation-roadmap.md | 403 +++++++ docs/concepts/technical-specification.md | 989 ++++++++++++++++++ docs/concepts/upstream-service-simulation.md | 393 +++++++ ...vice-lifecycle-and-power-on-enhancement.md | 391 +++++++ docs/device-lifecycle-summary.md | 150 +++ docs/diagrams/migration-flow.md | 222 ++++ docs/guides/IOT-IMPLEMENTATION-GUIDE.md | 753 +++++++++++++ docs/guides/MIGRATION-GUIDE.md | 436 ++++++++ docs/guides/MQTT-INTEGRATION-DESIGN.md | 764 ++++++++++++++ docs/images/README.md | 90 ++ docs/power-on-implementation-guide.md | 599 +++++++++++ docs/scmudc-events-analysis.md | 140 +++ 23 files changed, 6806 insertions(+), 99 deletions(-) create mode 100644 docs/SCMUDC-ENRICHMENT-IMPLEMENTATION.md create mode 100644 docs/analysis/IOT-CONFIG-SUMMARY.md create mode 100644 docs/analysis/IOT-CONFIGURATION-ANALYSIS.md create mode 100644 docs/concepts/README.md create mode 100644 docs/concepts/implementation-plan.md create mode 100644 docs/concepts/implementation-roadmap.md create mode 100644 docs/concepts/technical-specification.md create mode 100644 docs/concepts/upstream-service-simulation.md create mode 100644 docs/device-lifecycle-and-power-on-enhancement.md create mode 100644 docs/device-lifecycle-summary.md create mode 100644 docs/diagrams/migration-flow.md create mode 100644 docs/guides/IOT-IMPLEMENTATION-GUIDE.md create mode 100644 docs/guides/MIGRATION-GUIDE.md create mode 100644 docs/guides/MQTT-INTEGRATION-DESIGN.md create mode 100644 docs/images/README.md create mode 100644 docs/power-on-implementation-guide.md create mode 100644 docs/scmudc-events-analysis.md diff --git a/.github/markdown-link-check.json b/.github/markdown-link-check.json index b78d570..a824b80 100644 --- a/.github/markdown-link-check.json +++ b/.github/markdown-link-check.json @@ -25,6 +25,9 @@ }, { "pattern": "^https://pkg.go.dev.*badge" + }, + { + "pattern": "^\\.\\./images/(dashboard-home|account-creation|account-dashboard|usb-remote-services|device-discovery|device-registration|account-migration|migration-setup|migration-progress|migration-health|migration-complete|backup-setup)\\.png$" } ], "replacementPatterns": [ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5144bd1..7bd20a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,6 +144,29 @@ jobs: use-verbose-mode: "yes" config-file: ".github/markdown-link-check.json" + - name: Warn on pending images + run: | + IMAGES=( + "dashboard-home.png" + "account-creation.png" + "account-dashboard.png" + "usb-remote-services.png" + "device-discovery.png" + "device-registration.png" + "account-migration.png" + "migration-setup.png" + "migration-progress.png" + "migration-health.png" + "migration-complete.png" + "backup-setup.png" + ) + + for img in "${IMAGES[@]}"; do + if [ ! -f "docs/images/$img" ]; then + echo "::warning file=docs/guides/MIGRATION-GUIDE.md::Pending image '$img' is missing from docs/images/" + fi + done + - name: Validate API documentation run: | # Check that all documented endpoints exist in code diff --git a/docs/README.md b/docs/README.md index a7c17ca..5e8f21e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,33 +1,89 @@ # Bose SoundTouch Toolkit Documentation -Welcome to the documentation for the Bose SoundTouch Toolkit. This toolkit helps you keep your Bose SoundTouch speakers functional even after the Bose Cloud shutdown in May 2026. +Welcome to the documentation for the Bose SoundTouch Toolkit. This comprehensive toolkit helps you keep your Bose SoundTouch speakers functional even after the Bose Cloud shutdown in May 2026, with enhanced local management and monitoring capabilities. -## πŸ“– Quick Links +## πŸš€ Start Here -- [Cloud Shutdown Survival Guide](guides/SURVIVAL-GUIDE.md) -- [Migration & Safety Guide](guides/MIGRATION-SAFETY.md) -- [CLI Reference](guides/CLI-REFERENCE.md) -- [Getting Started](guides/GETTING-STARTED.md) -- [SoundTouch Service Guide](guides/SOUNDTOUCH-SERVICE.md) +### For New Users +- **[Complete Migration Guide](guides/MIGRATION-GUIDE.md)** - Step-by-step guide from Bose Cloud to local control +- **[Getting Started](guides/GETTING-STARTED.md)** - Quick introduction to the toolkit + +### For Existing Users +- **[Cloud Shutdown Survival Guide](guides/SURVIVAL-GUIDE.md)** - Prepare for the May 2026 shutdown +- **[SoundTouch Service Guide](guides/SOUNDTOUCH-SERVICE.md)** - Advanced service configuration + +## πŸ“‹ Essential Documentation + +The documentation is organized into three main categories: + +### 1. **User Guides** - For everyday users migrating and managing devices +### 2. **Technical Reference** - For developers and advanced configuration +### 3. **Concept Documentation** - For contributors and system architects ## πŸ—‚ Documentation Structure -### User Guides -- [Initial Device Setup](guides/DEVICE-INITIAL-SETUP.md) -- [HTTPS Setup](guides/HTTPS-SETUP.md) -- [Deployment Guide](guides/DEPLOYMENT.md) -- [Raspberry Pi Setup](guides/RASPBERRY-PI.md) -- [MAC Address Mapping](guides/MAC-ADDRESS-MAPPING.md) -- [Troubleshooting](guides/TROUBLESHOOTING.md) +## πŸ—‚ User Guides -### Technical Reference -- [API Endpoints](reference/API-ENDPOINTS.md) -- [WebSocket Events](reference/WEBSOCKET-EVENTS.md) -- [Zone Management](reference/ZONE-MANAGEMENT.md) -- [Preset Management](reference/PRESET-MANAGEMENT.md) +### Migration & Setup +- **[Complete Migration Guide](guides/MIGRATION-GUIDE.md)** - πŸ“– **Main guide** for migrating from Bose Cloud +- [Cloud Shutdown Survival Guide](guides/SURVIVAL-GUIDE.md) - Prepare for service shutdown +- [Migration & Safety Guide](guides/MIGRATION-SAFETY.md) - Advanced migration strategies +- [Initial Device Setup](guides/DEVICE-INITIAL-SETUP.md) - First-time device configuration +- [Raspberry Pi Setup](guides/RASPBERRY-PI.md) - Installing on Raspberry Pi + +### Daily Management +- [SoundTouch Service Guide](guides/SOUNDTOUCH-SERVICE.md) - Service operation and maintenance +- [Troubleshooting](guides/TROUBLESHOOTING.md) - Common issues and solutions +- [HTTPS Setup](guides/HTTPS-SETUP.md) - Secure connections +- [Deployment Guide](guides/DEPLOYMENT.md) - Production deployments + +### Advanced Features +- [MAC Address Mapping](guides/MAC-ADDRESS-MAPPING.md) - Device identification +- [CLI Reference](guides/CLI-REFERENCE.md) - Command-line tools +- [IoT Implementation Guide](guides/IOT-IMPLEMENTATION-GUIDE.md) - IoT integrations +- [MQTT Integration Design](guides/MQTT-INTEGRATION-DESIGN.md) - MQTT setup + +## πŸ“š Technical Reference + +### API Documentation +- [API Endpoints](reference/API-ENDPOINTS.md) - REST API reference +- [WebSocket Events](reference/WEBSOCKET-EVENTS.md) - Real-time events +- [Zone Management](reference/ZONE-MANAGEMENT.md) - Multi-room control +- [Preset Management](reference/PRESET-MANAGEMENT.md) - Preset operations ### Analysis & Research -- [Upstream URLs](analysis/UPSTREAM-URLS.md) -- [Device Redirect Methods](analysis/DEVICE-REDIRECT-METHODS.md) +- [Upstream URLs](analysis/UPSTREAM-URLS.md) - Bose service endpoints +- [Device Redirect Methods](analysis/DEVICE-REDIRECT-METHODS.md) - Migration techniques +- [IoT Configuration Analysis](analysis/IOT-CONFIGURATION-ANALYSIS.md) - Device configurations +- [IoT Config Summary](analysis/IOT-CONFIG-SUMMARY.md) - Configuration summaries + +### Device Lifecycle & Network Independence +- **[Device Lifecycle and /power_on Enhancement](device-lifecycle-and-power-on-enhancement.md)** - Complete analysis of device registration and network independence improvements +- [/power_on Implementation Guide](power-on-implementation-guide.md) - Technical implementation details for enhanced device management + +## πŸ— Concept Documentation + +### Enhanced Service Architecture +- **[Concept Overview](concepts/README.md)** - High-level architecture vision +- [Upstream Service Simulation](concepts/upstream-service-simulation.md) - Complete concept design +- [Implementation Plan](concepts/implementation-plan.md) - Development roadmap +- [Technical Specification](concepts/technical-specification.md) - Detailed specifications + +### Development Planning +- [Implementation Roadmap](concepts/implementation-roadmap.md) - Project phases and milestones + +## πŸ’‘ Quick Reference + +### Common Tasks +- **Migrate first device**: Follow [Migration Guide Step 5](guides/MIGRATION-GUIDE.md#step-5-migrate-individual-devices) +- **Check device health**: Dashboard β†’ Devices β†’ [Device Name] β†’ Health Status +- **Backup configuration**: Dashboard β†’ Settings β†’ Backup β†’ Create Backup +- **Add new device**: Dashboard β†’ Devices β†’ Discover Devices β†’ Register + +### Getting Help +- **Issues & Bugs**: [GitHub Issues](https://github.com/gesellix/Bose-SoundTouch/issues) +- **Questions & Discussion**: [GitHub Discussions](https://github.com/gesellix/Bose-SoundTouch/discussions) +- **Documentation**: Check troubleshooting guides first +- **Community**: Share experiences and help others For a complete list of all documents, see the [Summary](SUMMARY.md). diff --git a/docs/SCMUDC-ENRICHMENT-IMPLEMENTATION.md b/docs/SCMUDC-ENRICHMENT-IMPLEMENTATION.md new file mode 100644 index 0000000..3f9f82b --- /dev/null +++ b/docs/SCMUDC-ENRICHMENT-IMPLEMENTATION.md @@ -0,0 +1,190 @@ +# SCMUDC Enrichment Implementation Summary + +## Overview + +This document summarizes the implementation of SCMUDC (Sound Control Management Usage Data Collection) event enrichment in the AfterTouch toolkit. The enhancement provides human-readable analysis of device telemetry data to improve usability and debugging capabilities. + +## Problem Solved + +Previously, SCMUDC telemetry events were stored as raw JSON with Base64-encoded XML content, making them difficult to analyze. Users had to manually decode content to understand what device interactions were being recorded. + +## Solution Implemented + +### 1. Backend Enrichment (`pkg/service/proxy/`) + +#### New File: `scmudc.go` +- **SCMUDCRequest/SCMUDCEvent Structs**: Parse incoming telemetry JSON +- **EnrichedSCMUDCEvent Struct**: Human-readable analysis with decoded content +- **DecodedContent Struct**: Parsed XML metadata (track names, artwork URLs, etc.) +- **enrichSCMUDCRequest()**: Main enrichment function that: + - Identifies event origin (app, hardware, or internal system) + - Decodes Base64 XML content for device events + - Creates human-readable summaries +- **Helper Functions**: Button formatting, content summarization, origin descriptions + +#### Enhanced File: `recorder.go` +- **Updated save() method**: Extracts SCMUDC data during recording +- **New writeRequestWithEnrichment()**: Adds enriched comments to .http files +- **New writeResponseWithEnrichment()**: Includes SCMUDC analysis in response section +- **Updated Interaction struct**: Added `SCMUDCData` field for API responses +- **New extractSCMUDCFromFile()**: Parses enrichment data from existing .http files +- **Enhanced parseInteractionFile()**: Populates SCMUDC data when listing interactions + +### 2. Frontend Enhancement + +#### Updated HTML (`pkg/service/handlers/web/index.html`) +- **New Column**: Added "Event Details" to interactions table +- **Table Structure**: Updated to accommodate SCMUDC enrichment display + +#### Enhanced JavaScript (`pkg/service/handlers/web/js/script.js`) +- **Updated fetchInteractions()**: Displays enriched SCMUDC data with icons +- **New Helper Functions**: + - `getOriginIcon()`: Maps origins to emojis (πŸ“± App, πŸŽ›οΈ Hardware, πŸ”„ Internal) + - `getActionIcon()`: Maps actions to emojis (▢️ Play, ⏸️ Pause, etc.) + - `showSCMUDCDetails()`: Detailed popover for complex events + - `displaySCMUDCPopover()`: Modal dialog with full decoded content +- **Truncation Logic**: Long content shows "(...)" with click-to-expand + +## Event Origin Clarification + +Based on analysis of recorded data: + +| Origin | Source | Description | Example Events | +|--------|--------|-------------|----------------| +| `gabbo` | **SoundTouch App** | Mobile/desktop app UI interactions | Play, Pause, Power via app | +| `console` | **Device Hardware** | Physical buttons on speaker | Preset buttons, hardware power | +| `device` | **Internal System** | Automatic device responses | Content playback, system actions | + +## Enhanced .http File Format + +### Before (Raw) +```http +### POST /v1/scmudc/A81B6A536A98 +POST /v1/scmudc/A81B6A536A98 +Host: events.api.bosecm.com +... + +{"envelope":...,"payload":{"events":[{"data":{"contentItem":"PD94bWw..."}}]}} +``` + +### After (Enriched) +```http +### POST /v1/scmudc/A81B6A536A98 +// Origin: Internal System (device) +// Action: play-item +// Command: Billie Eilish - bad guy (instrumental version) +// Summary: Device: Spotify: Billie Eilish - bad guy (instrumental version) +// +// Decoded Content: +// - Source: SPOTIFY +// - Item: Billie Eilish - bad guy (instrumental version) +// - Account: gesellix +// - Artwork: https://i.scdn.co/image/ab67616d0000b273... +// +// Full XML Content: +// +// +// Billie Eilish - bad guy (instrumental version) +// https://i.scdn.co/image/ab67616d0000b273... +// +POST /v1/scmudc/A81B6A536A98 +... + +> {% + // Response: 200 OK + // SCMUDC Event Analysis: + // - Origin: Internal System (device) + // - Action: play-item + // - Summary: Device: Spotify: Billie Eilish - bad guy (instrumental version) + // - Content: Billie Eilish - bad guy (instrumental version) + // - Account: gesellix +%} +``` + +## Web UI Enhancement + +### Interactions Table +- **New Column**: "Event Details" shows enriched summaries +- **Visual Icons**: Origin and action type indicators +- **Truncation**: Long content abbreviated with "(...)" expansion +- **Backward Compatibility**: Works with existing recordings + +### Event Details Display +``` +πŸ“± ▢️ Play Button (Simple app action) +πŸ”„ 🎡 Billie Eilish - bad guy... (...) (Complex device event with details) +πŸŽ›οΈ ⭐ Preset 5 (Hardware preset button) +``` + +### Detailed Popover +For complex events, clicking "(...)" shows: +- **Origin Description**: "SoundTouch App" instead of "gabbo" +- **Full Content Information**: Track names, artwork URLs, account details +- **Complete XML**: Formatted and readable content item data + +## Implementation Benefits + +### For Users +- **Immediate Recognition**: See what actions were performed without decoding +- **Better Debugging**: Quick identification of app vs. hardware vs. system events +- **Rich Context**: Track names, accounts, and content sources visible at a glance + +### For Developers +- **Structured Data**: Consistent parsing and enrichment pipeline +- **Extensible**: Easy to add new event types and origins +- **Backward Compatible**: Existing recordings work without re-processing + +### For Analysis +- **Pattern Recognition**: Quickly identify user behavior patterns +- **Service Integration**: See which music services are being used +- **Device Usage**: Understand app vs. hardware control preferences + +## File Structure + +``` +pkg/service/proxy/ +β”œβ”€β”€ scmudc.go # New: SCMUDC enrichment logic +β”œβ”€β”€ recorder.go # Enhanced: Enrichment integration +β”‚ +pkg/service/handlers/web/ +β”œβ”€β”€ index.html # Enhanced: New table column +β”œβ”€β”€ js/script.js # Enhanced: SCMUDC display logic +β”‚ +docs/ +β”œβ”€β”€ scmudc-events-analysis.md # New: Analysis documentation +β”œβ”€β”€ SCMUDC-ENRICHMENT-IMPLEMENTATION.md # This file +``` + +## Technical Decisions + +### Base64 Decoding Strategy +- **When**: During recording (not on-demand) for performance +- **Fallback**: Parse from .http files if enrichment missing +- **Storage**: Both enriched comments and structured data in API responses + +### Icon Selection +- **Emoji Usage**: Universal, colorful, intuitive recognition +- **Semantic Mapping**: Icons match function (πŸ“± for app, πŸŽ›οΈ for hardware) +- **Fallback**: Generic icons (❓, πŸ”˜) for unknown types + +### Backward Compatibility +- **Graceful Degradation**: Missing enrichment data doesn't break UI +- **File Parsing**: Extract enrichment from existing .http files +- **API Enhancement**: New fields optional in Interaction struct + +## Future Enhancement Opportunities + +1. **Event Correlation**: Link device events to user actions +2. **Statistics Dashboard**: Origin-based usage analytics +3. **Content Recommendations**: Track listening patterns +4. **Device Health**: Monitor interaction frequency and patterns +5. **Export Features**: CSV/JSON export of enriched event data + +## Testing Considerations + +- **Edge Cases**: Malformed Base64, missing XML elements +- **Performance**: Large numbers of SCMUDC events +- **Browser Compatibility**: Emoji display across different browsers +- **Data Validation**: Ensure enrichment doesn't introduce errors + +This implementation significantly improves the usability of SCMUDC telemetry data while maintaining full backward compatibility and raw data access for advanced users. \ No newline at end of file diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index d750253..b87a25b 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -14,6 +14,9 @@ * [Deployment](guides/DEPLOYMENT.md) * [Raspberry Pi Guide](guides/RASPBERRY-PI.md) * [Troubleshooting](guides/TROUBLESHOOTING.md) +* [IoT Implementation Guide](guides/IOT-IMPLEMENTATION-GUIDE.md) +* [Migration Guide](guides/MIGRATION-GUIDE.md) +* [MQTT Integration Design](guides/MQTT-INTEGRATION-DESIGN.md) * [Useful Links](#useful-links) ### Useful Links @@ -51,6 +54,8 @@ * [Anonymization Summary](analysis/ANONYMIZATION-SUMMARY.md) * [Device Redirect Methods](analysis/DEVICE-REDIRECT-METHODS.md) * [Wiki API Comparison](analysis/WIKI-COMPARISON.md) +* [IoT Config Summary](analysis/IOT-CONFIG-SUMMARY.md) +* [IoT Configuration Analysis](analysis/IOT-CONFIGURATION-ANALYSIS.md) ## Appendix (Other Documents) * [API Navigation Reference](API-NAVIGATION-REFERENCE.md) @@ -70,3 +75,8 @@ * [Undocumented Community Features](UNDOCUMENTED-COMMUNITY-FEATURES.md) * [Unimplemented Endpoints](UNIMPLEMENTED-ENDPOINTS.md) * [Preset Store](preset-store.md) +* [SCMUDC Enrichment Implementation](SCMUDC-ENRICHMENT-IMPLEMENTATION.md) +* [Device Lifecycle and Power On Enhancement](device-lifecycle-and-power-on-enhancement.md) +* [Device Lifecycle Summary](device-lifecycle-summary.md) +* [Power On Implementation Guide](power-on-implementation-guide.md) +* [SCMUDC Events Analysis](scmudc-events-analysis.md) diff --git a/docs/analysis/IOT-CONFIG-SUMMARY.md b/docs/analysis/IOT-CONFIG-SUMMARY.md new file mode 100644 index 0000000..bf8322b --- /dev/null +++ b/docs/analysis/IOT-CONFIG-SUMMARY.md @@ -0,0 +1,189 @@ +# IoT Configuration Quick Reference + +## Key Files and Locations + +| File/Location | Purpose | Notes | +|-----------------------------------------|------------------------|-----------------------------------------| +| `/mnt/nv/BoseApp-Persistence/1/IoT.xml` | Main IoT configuration | Contains clientID, endpoint, deployment | +| `/opt/Bose/IoT` | IoT service binary | ARM executable, AWS IoT SDK | +| `/mnt/nv/IoTCerts/` | Certificate storage | Device certs and private keys | +| `/etc/init.d/SoundTouch` | System startup script | Creates directory structure | +| `/opt/Bose/etc/Shepherd-noncore.xml` | Service configuration | Defines IoT daemon startup | + +## Configuration Parameters + +### IoT.xml Structure +```xml + +``` + +### Device-Specific Values +- **ST20**: `clientID="577ecfcc-2db3-4989-92c9-76d7704f9fb3"` +- **ST10**: `clientID="eb1a6d8f-0bb1-4aa7-9113-ea673fcef96e"` +- **Endpoint**: `a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com` (XML) +- **Backup Endpoint**: `amqmidtcohfms.iot.us-east-1.amazonaws.com` (hardcoded) + +## Protocol Stack + +``` +Application Layer: AWS IoT Device Shadows (JSON) +Presentation Layer: RapidJSON parsing/serialization +Session Layer: MQTT v3.1.1 +Transport Layer: TLS v1.2 +Network Layer: TCP/IP +``` + +## Certificate Files + +| File | Location | Purpose | +|-----------------------|---------------------|---------------------------| +| `iot-cert.pem.crt` | `/mnt/nv/IoTCerts/` | Device client certificate | +| `iot-private.pem.key` | `/mnt/nv/IoTCerts/` | Device private key | +| `rootCA.crt` | `/var/lib/iot/` | AWS IoT Root CA | + +## MQTT Topics + +### Shadow Operations +``` +$aws/things/{clientID}/shadow/update +$aws/things/{clientID}/shadow/update/accepted +$aws/things/{clientID}/shadow/update/rejected +$aws/things/{clientID}/shadow/delete +``` + +### JSON Payload Examples + +#### Device State Report +```json +{ + "state": { + "reported": { + "deviceState": "CONNECTED", + "powerState": "ON", + "zoneState": "...", + "groupState": "..." + } + } +} +``` + +#### Disconnection Message +```json +{ + "state": { + "reported": { + "deviceState": "DISCONNECTED" + } + } +} +``` + +## Process Information + +- **IoT Service PID**: 1837 +- **BoseApp PID**: 1846 +- **Daemon Manager**: Shepherd +- **Service Type**: Non-core (stopped during updates) + +## Registration Flow + +1. Device generates X.509 CSR +2. Calls `https://voice.api.bose.io/alexa/certificate` +3. Receives device certificate +4. Stores cert/key in `/mnt/nv/IoTCerts/` +5. Connects to AWS IoT using certificate auth + +## Directory Creation (Init Script) + +```bash +mkdir -p /mnt/nv/BoseLog /mnt/nv/IoTCerts /mnt/nv/BoseApp-Persistence/1 +mkdir -m 700 -p /mnt/nv/BoseApp-Persistence/1/Keys +``` + +## Error Messages and Debugging + +### Common Log Messages +- `"Connection attempt %u to MQTT port at host %s"` +- `"MQTT port not available. Retrying in %u seconds"` +- `"Device connected with MQTT"` +- `"got shadow response: accepted. Payload: %s"` +- `"Failed to register device and get certificate, retrying"` + +### Connection States +- `"MQTT port is open"` +- `"Successfully connected to MQTT server"` +- `"Disconnecting from IoT server"` +- `"UpdateShadow called when network is not ready"` + +## Integration Points + +### AWS Services +- AWS IoT Core (MQTT broker) +- AWS IoT Device Management (certificates) +- AWS IoT Device Shadows (state sync) + +### Bose Ecosystem +- Mobile apps (remote control) +- Alexa integration (voice commands) +- Multi-room audio (zone coordination) +- OTA updates (firmware management) + +## Quick Troubleshooting + +1. **No IoT connectivity**: Check certificate files in `/mnt/nv/IoTCerts/` +2. **Certificate errors**: Verify registration endpoint accessibility +3. **MQTT failures**: Check both primary and backup endpoints +4. **Config issues**: Validate IoT.xml format and clientID uniqueness +5. **Service not starting**: Check Shepherd configuration and process status + +## MQTT Monitoring Capabilities + +### Direct Access with Device Credentials +```bash +# Subscribe to device shadow events (own device only) +mosquitto_sub -h a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com \ + -p 8883 --cafile /var/lib/iot/rootCA.crt \ + --cert /mnt/nv/IoTCerts/iot-cert.pem.crt \ + --key /mnt/nv/IoTCerts/iot-private.pem.key \ + -t '$aws/things/577ecfcc-2db3-4989-92c9-76d7704f9fb3/shadow/#' +``` + +### AWS IoT Policy Restrictions +- Device certificates limited to own clientID topics only +- No wildcard subscriptions across devices +- IP/location restrictions may apply +- Certificate revocation for unusual activity + +### Alternative Monitoring Methods +```bash +# Network traffic capture (less intrusive) +tcpdump -i eth0 -s0 -w soundtouch_iot.pcap host a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com + +# Monitor connection patterns +tcpdump -i eth0 -n "host a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com and port 8883" +``` + +### Expected Message Examples +```json +// Power state change +{"state":{"reported":{"powerState":"ON","deviceState":"CONNECTED"}}} + +// Volume adjustment +{"state":{"reported":{"volume":25,"muted":false}}} + +// Zone configuration +{"state":{"reported":{"zoneState":"master","groupMembers":["device1"]}}} +``` + +## Security Notes + +- TLS 1.2 encryption for all communications +- X.509 mutual authentication +- Private keys stored with 700 permissions +- No hardcoded credentials in binaries +- Automatic certificate lifecycle management +- **Monitoring Constraints**: Device credentials restricted to own device topics +- **Ethical Consideration**: Only monitor devices you own diff --git a/docs/analysis/IOT-CONFIGURATION-ANALYSIS.md b/docs/analysis/IOT-CONFIGURATION-ANALYSIS.md new file mode 100644 index 0000000..0b515df --- /dev/null +++ b/docs/analysis/IOT-CONFIGURATION-ANALYSIS.md @@ -0,0 +1,370 @@ +# IoT Configuration Analysis + +## Overview + +This document provides a detailed analysis of the AWS IoT configuration system used by Bose SoundTouch devices, based on firmware backup analysis from ST10 and ST20 models. + +## Configuration Files + +### IoT.xml Location and Content + +The IoT configuration is stored in XML format at: +- **Path**: `/mnt/nv/BoseApp-Persistence/1/IoT.xml` +- **Purpose**: Contains AWS IoT Core connection parameters + +#### ST20 Configuration +```xml + + +``` + +#### ST10 Configuration +```xml + + +``` + +### Key Observations +- Each device has a unique `clientID` (UUID format) +- Both devices use the same AWS IoT endpoint +- Both are configured for production deployment (`PROD`) + +## Binary Analysis + +### Primary IoT Service Binary + +**Location**: `/opt/Bose/IoT` +- **Type**: ARM ELF 32-bit executable +- **Purpose**: Main IoT daemon process +- **Framework**: AWS IoT SDK for C++ + +### Certificate and Key Management + +The IoT binary manages the following certificate files: + +| File | Location | Purpose | +|-----------------------|---------------------|-----------------------------| +| `iot-cert.pem.crt` | `/mnt/nv/IoTCerts/` | Device client certificate | +| `iot-private.pem.key` | `/mnt/nv/IoTCerts/` | Device private key | +| `rootCA.crt` | `/var/lib/iot/` | AWS IoT Root CA certificate | + +### Certificate Registration Process + +1. **CSR Generation**: Device generates X.509 certificate signing request +2. **Registration Endpoint**: `https://voice.api.bose.io/alexa/certificate` +3. **Certificate Storage**: Certificates stored in `/mnt/nv/IoTCerts/` +4. **Automatic Provisioning**: Process appears to be automated during device setup + +## Protocol Analysis + +### Connection Details + +- **Protocol**: MQTT over TLS 1.2 +- **Port**: Standard MQTT over SSL (likely 8883) +- **Authentication**: X.509 client certificate mutual authentication +- **Endpoint Redundancy**: + - Primary (hardcoded): `amqmidtcohfms.iot.us-east-1.amazonaws.com` + - Fallback (XML config): `a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com` + +### AWS IoT Device Shadow Integration + +The system uses AWS IoT Device Shadows for state management: + +#### Topic Structure +``` +$aws/things/{thing_name}/shadow/update +$aws/things/{thing_name}/shadow/update/accepted +$aws/things/{thing_name}/shadow/update/rejected +$aws/things/{thing_name}/shadow/delete +``` + +#### Shadow JSON Format +```json +{ + "state": { + "desired": {}, + "reported": { + "deviceState": "CONNECTED|DISCONNECTED", + "powerState": "ON|OFF", + "zoneState": "...", + "groupState": "..." + } + }, + "version": 0, + "clientToken": "...", + "timestamp": 0 +} +``` + +### Message Types + +1. **Device State Updates** + - Connection status (`CONNECTED`/`DISCONNECTED`) + - Power state changes + - Audio zone configuration + - Multi-room grouping status + +2. **Shadow Delta Processing** + - Receives desired state changes + - Updates device configuration + - Reports new state back to shadow + +## System Integration + +### Service Management + +The IoT service is managed by the Shepherd daemon system: + +**Configuration**: `/opt/Bose/etc/Shepherd-noncore.xml` +```xml + + + + + -c + /opt/Bose/etc/Voice.xml + + +``` + +### Directory Structure Creation + +The SoundTouch init script (`/etc/init.d/SoundTouch`) ensures proper directory structure: + +```bash +mkdir -p /mnt/nv/BoseLog /mnt/nv/IoTCerts /mnt/nv/BoseApp-Persistence/1 +mkdir -m 700 -p /mnt/nv/BoseApp-Persistence/1/Keys +``` + +### Process Information + +From runtime analysis (`/var/run/shepherd/pids`): +- IoT service runs as PID 1837 +- BoseApp service runs as PID 1846 +- Both services are active during normal operation + +## Configuration Dependencies + +### Files That Reference IoT Configuration + +1. **IoT Binary** (`/opt/Bose/IoT`) + - Primary consumer of IoT.xml configuration + - Contains hardcoded backup endpoints + - Manages certificate lifecycle + +2. **BoseApp Binary** (`/opt/Bose/BoseApp`) + - References BoseApp-Persistence directory structure + - May trigger IoT updates based on device state changes + +3. **SoundTouch Init Script** (`/etc/init.d/SoundTouch`) + - Creates necessary directory structure + - Ensures proper permissions for certificate storage + +4. **Shepherd Configuration** (`/opt/Bose/etc/Shepherd-noncore.xml`) + - Defines IoT service startup parameters + - Manages service lifecycle + +## Security Considerations + +### Certificate Management +- Private keys stored with 700 permissions +- Certificates managed automatically by the device +- Registration process appears to use device-specific authentication + +### Network Security +- All communication over TLS 1.2 +- Mutual authentication using X.509 certificates +- AWS IoT Core provides additional access controls + +### Configuration Protection +- Configuration files stored in persistent storage +- Directory structure created with appropriate permissions +- No hardcoded credentials in binaries (uses certificate-based auth) + +## Integration Points + +### AWS Services +- **AWS IoT Core**: Primary messaging and device management +- **AWS IoT Device Management**: Certificate provisioning +- **AWS IoT Device Shadows**: State synchronization + +### Bose Services +- **Mobile Applications**: Remote control and monitoring +- **Alexa Integration**: Voice control capabilities +- **Multi-room Audio**: Zone and group coordination + +### Device Functions +- **Power Management**: Remote power on/off +- **Audio Control**: Volume, source selection +- **Network Configuration**: WiFi and connectivity settings +- **Firmware Updates**: OTA update coordination + +## Troubleshooting + +### Common Issues + +1. **Certificate Problems** + - Check `/mnt/nv/IoTCerts/` for valid certificates + - Verify certificate registration endpoint accessibility + - Ensure proper file permissions (600 for keys) + +2. **Connection Issues** + - Verify both primary and fallback endpoints + - Check TLS 1.2 support and cipher suites + - Validate clientID uniqueness + +3. **Configuration Issues** + - Ensure IoT.xml has proper XML format + - Verify clientID is valid UUID format + - Check deployment parameter matches environment + +### Debug Information + +The IoT binary provides extensive logging for: +- MQTT connection attempts and status +- Certificate loading and validation +- Shadow message processing +- Network state changes + +## MQTT Monitoring and Security Considerations + +### Direct MQTT Access with Device Credentials + +With access to the device's private key and certificate, it's technically possible to subscribe to MQTT events: + +```bash +# Subscribe to device shadow events +mosquitto_sub -h a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com \ + -p 8883 --cafile /var/lib/iot/rootCA.crt \ + --cert /mnt/nv/IoTCerts/iot-cert.pem.crt \ + --key /mnt/nv/IoTCerts/iot-private.pem.key \ + -t '$aws/things/_uuid_/shadow/#' +``` + +### Security Constraints and Limitations + +#### AWS IoT Policy Restrictions +Device certificates are bound to specific policies that typically restrict: +- Access to device-specific topics only (`$aws/things/{clientID}/shadow/*`) +- No wildcard subscriptions across multiple devices +- Limited publish/subscribe permissions +- Possible IP geolocation restrictions + +#### Example Policy Structure +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "iot:Connect", + "Resource": "arn:aws:iot:us-east-1:*:client/${iot:ClientId}" + }, + { + "Effect": "Allow", + "Action": ["iot:Publish", "iot:Subscribe", "iot:Receive"], + "Resource": [ + "arn:aws:iot:us-east-1:*:topic/$aws/things/${iot:ClientId}/shadow/*", + "arn:aws:iot:us-east-1:*:topicfilter/$aws/things/${iot:ClientId}/shadow/*" + ] + } + ] +} +``` + +#### Additional Security Measures +- Certificate revocation for unusual activity +- Device fingerprinting and connection frequency limits +- Service shutdown timeline (May 2026) affecting endpoint availability + +### Alternative Monitoring Approaches + +#### Network Traffic Capture +A less intrusive method to analyze MQTT communication patterns: + +```bash +# Capture encrypted MQTT traffic from the actual device +tcpdump -i eth0 -s0 -w soundtouch_iot.pcap host a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com + +# Monitor connection patterns +tcpdump -i eth0 -n "host a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com and port 8883" +``` + +#### Local MQTT Broker Setup +For development and testing, create a local MQTT broker that mimics AWS IoT behavior: + +```bash +# Install and configure Mosquitto +sudo apt-get install mosquitto mosquitto-clients + +# Create test shadow topics +mosquitto_pub -h localhost -t '$aws/things/test-device/shadow/update' \ + -m '{"state":{"reported":{"deviceState":"CONNECTED"}}}' +``` + +### Ethical and Legal Considerations + +- **Device Ownership**: Only monitor devices you own +- **Terms of Service**: Using credentials outside device context may violate Bose ToS +- **Unauthorized Access**: Accessing Bose's AWS infrastructure could be considered inappropriate +- **Research Purpose**: Limit monitoring to understanding message formats for local alternatives + +### Expected Message Examples + +If monitoring is successful, typical shadow messages include: + +```json +// Power state change +{ + "state": { + "reported": { + "powerState": "ON", + "deviceState": "CONNECTED", + "timestamp": 1703875200 + } + } +} + +// Volume adjustment +{ + "state": { + "reported": { + "volume": 25, + "muted": false + } + } +} + +// Zone configuration +{ + "state": { + "reported": { + "zoneState": "master", + "groupMembers": ["device1", "device2"] + } + } +} +``` + +### Recommended Research Approach + +1. **Document Message Formats**: Capture and analyze JSON structures +2. **Understand State Transitions**: Map device actions to shadow updates +3. **Build Local Alternative**: Use insights to create local MQTT shadow service +4. **Prepare for Service Shutdown**: Develop migration strategy before May 2026 + +## Conclusion + +The Bose SoundTouch IoT configuration system is a sophisticated implementation using AWS IoT Core for real-time device management. The system provides: + +- Secure, certificate-based authentication +- Reliable bi-directional communication +- Comprehensive device state management +- Integration with voice assistants and mobile applications +- Robust error handling and retry mechanisms + +This architecture enables seamless remote control, monitoring, and coordination of SoundTouch devices across multiple platforms and services. diff --git a/docs/analysis/UPSTREAM-URLS.md b/docs/analysis/UPSTREAM-URLS.md index 94ecb70..8198b2b 100644 --- a/docs/analysis/UPSTREAM-URLS.md +++ b/docs/analysis/UPSTREAM-URLS.md @@ -6,13 +6,13 @@ This document provides a comprehensive overview of the upstream Bose cloud servi SoundTouch devices use a set of primary domains for their operation. These are often configurable via the `SoundTouchSdkPrivateCfg.xml` file. -| Service | Primary Domain | Purpose | -| :--- | :--- | :--- | -| **Marge** | `streaming.bose.com` | Account management, streaming source providers, and preset sync. | -| **BMX Registry** | `content.api.bose.io` | Bose Media eXchange service discovery and registry. | -| **Stats/Analytics** | `events.api.bosecm.com` | Telemetry, device events, and usage statistics. | -| **Software Update** | `worldwide.bose.com` | Firmware update checks and downloads (path: `/updates/soundtouch`). | -| **Voice/Alexa** | `voice.api.bose.io` | Token management for Amazon Alexa integration. | +| Service | Primary Domain | Purpose | +|:--------------------|:------------------------|:--------------------------------------------------------------------| +| **Marge** | `streaming.bose.com` | Account management, streaming source providers, and preset sync. | +| **BMX Registry** | `content.api.bose.io` | Bose Media eXchange service discovery and registry. | +| **Stats/Analytics** | `events.api.bosecm.com` | Telemetry, device events, and usage statistics. | +| **Software Update** | `worldwide.bose.com` | Firmware update checks and downloads (path: `/updates/soundtouch`). | +| **Voice/Alexa** | `voice.api.bose.io` | Token management for Amazon Alexa integration. | ## Internal & Development Domains diff --git a/docs/analysis/WIKI-COMPARISON.md b/docs/analysis/WIKI-COMPARISON.md index 663bc27..ab5514f 100644 --- a/docs/analysis/WIKI-COMPARISON.md +++ b/docs/analysis/WIKI-COMPARISON.md @@ -1,7 +1,7 @@ # SoundTouch API Comparison: Community Wiki vs Current Implementation **Date:** January 2026 -**Source:** [SoundTouch Plus Wiki](https://github.com/thlucas1/homeassistantcomponent_soundtouchplus/wiki/SoundTouch-WebServices-API) +**Source:** [SoundTouch Plus Wiki](https://github.com/thlucas1/homeassistantcomponent_soundtouchplus/wiki/SoundTouch-WebServices-API) **Our Implementation:** Bose-SoundTouch Go Library v1.0 ## Executive Summary @@ -20,84 +20,84 @@ The SoundTouch Plus community wiki documents **87 distinct API endpoints** with ### βœ… Already Implemented (23 endpoints) -| Endpoint | Wiki Status | Our Status | Notes | -|----------|-------------|------------|-------| -| `/info` | βœ… Documented | βœ… Complete | Device information | -| `/now_playing` | βœ… Documented | βœ… Complete | Current playback status | -| `/key` | βœ… Documented | βœ… Complete | Key press/release simulation | -| `/volume` | βœ… Documented | βœ… Complete | Volume and mute control | -| `/bass` | βœ… Documented | βœ… Complete | Bass level control | -| `/bassCapabilities` | βœ… Documented | βœ… Complete | Bass capability detection | -| `/sources` | βœ… Documented | βœ… Complete | Available audio sources | -| `/select` | βœ… Documented | βœ… Complete | Source selection | -| `/presets` | βœ… Documented | βœ… Complete | Preset configurations (read-only) | -| `/getZone` | βœ… Documented | βœ… Complete | Zone status and membership | -| `/setZone` | βœ… Documented | βœ… Complete | Zone creation and management | -| `/addZoneSlave` | βœ… Documented | βœ… Complete | Add device to zone | -| `/removeZoneSlave` | βœ… Documented | βœ… Complete | Remove device from zone | -| `/capabilities` | βœ… Documented | βœ… Complete | Device feature capabilities | -| `/audiodspcontrols` | βœ… Documented | βœ… Complete | Audio DSP modes and video sync | -| `/audioproducttonecontrols` | βœ… Documented | βœ… Complete | Advanced bass/treble controls | -| `/audioproductlevelcontrols` | βœ… Documented | βœ… Complete | Speaker level controls | -| `/name` (GET/POST) | βœ… Documented | βœ… Complete | Device name management | -| `/balance` | βœ… Documented | βœ… Complete | Stereo balance control | -| `/clockTime` | βœ… Documented | βœ… Complete | Device time management | -| `/clockDisplay` | βœ… Documented | βœ… Complete | Clock display settings | -| `/networkInfo` | βœ… Documented | βœ… Complete | Network connectivity info | -| `/requestToken` | βœ… Documented | βœ… Complete | Bearer token generation | +| Endpoint | Wiki Status | Our Status | Notes | +|------------------------------|--------------|------------|-----------------------------------| +| `/info` | βœ… Documented | βœ… Complete | Device information | +| `/now_playing` | βœ… Documented | βœ… Complete | Current playback status | +| `/key` | βœ… Documented | βœ… Complete | Key press/release simulation | +| `/volume` | βœ… Documented | βœ… Complete | Volume and mute control | +| `/bass` | βœ… Documented | βœ… Complete | Bass level control | +| `/bassCapabilities` | βœ… Documented | βœ… Complete | Bass capability detection | +| `/sources` | βœ… Documented | βœ… Complete | Available audio sources | +| `/select` | βœ… Documented | βœ… Complete | Source selection | +| `/presets` | βœ… Documented | βœ… Complete | Preset configurations (read-only) | +| `/getZone` | βœ… Documented | βœ… Complete | Zone status and membership | +| `/setZone` | βœ… Documented | βœ… Complete | Zone creation and management | +| `/addZoneSlave` | βœ… Documented | βœ… Complete | Add device to zone | +| `/removeZoneSlave` | βœ… Documented | βœ… Complete | Remove device from zone | +| `/capabilities` | βœ… Documented | βœ… Complete | Device feature capabilities | +| `/audiodspcontrols` | βœ… Documented | βœ… Complete | Audio DSP modes and video sync | +| `/audioproducttonecontrols` | βœ… Documented | βœ… Complete | Advanced bass/treble controls | +| `/audioproductlevelcontrols` | βœ… Documented | βœ… Complete | Speaker level controls | +| `/name` (GET/POST) | βœ… Documented | βœ… Complete | Device name management | +| `/balance` | βœ… Documented | βœ… Complete | Stereo balance control | +| `/clockTime` | βœ… Documented | βœ… Complete | Device time management | +| `/clockDisplay` | βœ… Documented | βœ… Complete | Clock display settings | +| `/networkInfo` | βœ… Documented | βœ… Complete | Network connectivity info | +| `/requestToken` | βœ… Documented | βœ… Complete | Bearer token generation | ### πŸ”₯ High Priority Missing (20 endpoints) -| Endpoint | Wiki Status | Priority | Use Case | -|----------|-------------|----------|----------| -| `/storePreset` | βœ… Detailed | **HIGH** | Save stations/playlists to presets | -| `/removePreset` | βœ… Detailed | **HIGH** | Delete saved presets | -| `/selectPreset` | βœ… Detailed | **HIGH** | Play preset by ID | -| `/setMusicServiceAccount` | βœ… Detailed | **HIGH** | Add Spotify/Pandora accounts | -| `/removeMusicServiceAccount` | βœ… Detailed | **HIGH** | Remove music service accounts | -| `/searchStation` | βœ… Detailed | **HIGH** | Find Pandora/Spotify content | -| `/addStation` | βœ… Detailed | **HIGH** | Add stations to favorites | -| `/removeStation` | βœ… Detailed | **HIGH** | Remove stations from favorites | -| `/navigate` | βœ… Detailed | **HIGH** | Browse music libraries/services | -| `/search` | βœ… Detailed | **HIGH** | Search music content | -| `/userPlayControl` | βœ… Detailed | **HIGH** | Play/pause/stop controls | -| `/userRating` | βœ… Detailed | **HIGH** | Thumbs up/down ratings | -| `/recents` | βœ… Detailed | **HIGH** | Recently played content | -| `/standby` | βœ… Detailed | **HIGH** | Power management | -| `/powerManagement` | βœ… Detailed | **HIGH** | Power state information | -| `/lowPowerStandby` | βœ… Detailed | **HIGH** | Low-power mode | -| `/listMediaServers` | βœ… Detailed | **HIGH** | UPnP/DLNA server discovery | -| `/serviceAvailability` | βœ… Detailed | **HIGH** | Source availability status | -| `/introspect` | βœ… Detailed | **HIGH** | Music service account status | -| `/language` | βœ… Detailed | **HIGH** | Device language settings | +| Endpoint | Wiki Status | Priority | Use Case | +|------------------------------|-------------|----------|------------------------------------| +| `/storePreset` | βœ… Detailed | **HIGH** | Save stations/playlists to presets | +| `/removePreset` | βœ… Detailed | **HIGH** | Delete saved presets | +| `/selectPreset` | βœ… Detailed | **HIGH** | Play preset by ID | +| `/setMusicServiceAccount` | βœ… Detailed | **HIGH** | Add Spotify/Pandora accounts | +| `/removeMusicServiceAccount` | βœ… Detailed | **HIGH** | Remove music service accounts | +| `/searchStation` | βœ… Detailed | **HIGH** | Find Pandora/Spotify content | +| `/addStation` | βœ… Detailed | **HIGH** | Add stations to favorites | +| `/removeStation` | βœ… Detailed | **HIGH** | Remove stations from favorites | +| `/navigate` | βœ… Detailed | **HIGH** | Browse music libraries/services | +| `/search` | βœ… Detailed | **HIGH** | Search music content | +| `/userPlayControl` | βœ… Detailed | **HIGH** | Play/pause/stop controls | +| `/userRating` | βœ… Detailed | **HIGH** | Thumbs up/down ratings | +| `/recents` | βœ… Detailed | **HIGH** | Recently played content | +| `/standby` | βœ… Detailed | **HIGH** | Power management | +| `/powerManagement` | βœ… Detailed | **HIGH** | Power state information | +| `/lowPowerStandby` | βœ… Detailed | **HIGH** | Low-power mode | +| `/listMediaServers` | βœ… Detailed | **HIGH** | UPnP/DLNA server discovery | +| `/serviceAvailability` | βœ… Detailed | **HIGH** | Source availability status | +| `/introspect` | βœ… Detailed | **HIGH** | Music service account status | +| `/language` | βœ… Detailed | **HIGH** | Device language settings | ### 🎡 Music Service Management (12 endpoints) -| Category | Endpoints | Wiki Coverage | Notes | -|----------|-----------|---------------|-------| -| **Account Management** | `/setMusicServiceAccount`, `/removeMusicServiceAccount` | βœ… Full XML examples | Pandora, Spotify, NAS setup | -| **Station Management** | `/searchStation`, `/addStation`, `/removeStation` | βœ… Pandora tested | Station discovery and favorites | -| **Content Navigation** | `/navigate`, `/search` | βœ… Detailed examples | Music library browsing | -| **Track Information** | `/trackInfo`, `/introspect` | βœ… Service-specific | Extended metadata | +| Category | Endpoints | Wiki Coverage | Notes | +|------------------------|---------------------------------------------------------|---------------------|---------------------------------| +| **Account Management** | `/setMusicServiceAccount`, `/removeMusicServiceAccount` | βœ… Full XML examples | Pandora, Spotify, NAS setup | +| **Station Management** | `/searchStation`, `/addStation`, `/removeStation` | βœ… Pandora tested | Station discovery and favorites | +| **Content Navigation** | `/navigate`, `/search` | βœ… Detailed examples | Music library browsing | +| **Track Information** | `/trackInfo`, `/introspect` | βœ… Service-specific | Extended metadata | ### 🏠 Smart Home Integration (15 endpoints) -| Category | Endpoints | Wiki Coverage | Notes | -|----------|-----------|---------------|-------| -| **Notifications** | `/speaker`, `/playNotification` | βœ… TTS examples | Text-to-speech, URL playback | -| **Power Management** | `/standby`, `/powerManagement`, `/lowPowerStandby` | βœ… Complete | Smart home automation | -| **Network Management** | `/performWirelessSiteSurvey`, `/addWirelessProfile`, `/getActiveWirelessProfile` | βœ… WiFi setup | Network configuration | -| **Bluetooth** | `/enterBluetoothPairing`, `/clearBluetoothPaired`, `/bluetoothInfo` | βœ… Pairing control | Bluetooth management | -| **Source Control** | `/selectLastSource`, `/selectLastSoundTouchSource`, `/selectLocalSource` | βœ… Source switching | Quick source access | +| Category | Endpoints | Wiki Coverage | Notes | +|------------------------|----------------------------------------------------------------------------------|--------------------|------------------------------| +| **Notifications** | `/speaker`, `/playNotification` | βœ… TTS examples | Text-to-speech, URL playback | +| **Power Management** | `/standby`, `/powerManagement`, `/lowPowerStandby` | βœ… Complete | Smart home automation | +| **Network Management** | `/performWirelessSiteSurvey`, `/addWirelessProfile`, `/getActiveWirelessProfile` | βœ… WiFi setup | Network configuration | +| **Bluetooth** | `/enterBluetoothPairing`, `/clearBluetoothPaired`, `/bluetoothInfo` | βœ… Pairing control | Bluetooth management | +| **Source Control** | `/selectLastSource`, `/selectLastSoundTouchSource`, `/selectLocalSource` | βœ… Source switching | Quick source access | ### πŸ“± Advanced Device Features (19 endpoints) -| Category | Endpoints | Wiki Coverage | Notes | -|----------|-----------|---------------|-------| -| **Stereo Pairs** | `/getGroup`, `/addGroup`, `/removeGroup`, `/updateGroup` | βœ… ST-10 specific | L/R speaker pairing | -| **System Info** | `/soundTouchConfigurationStatus`, `/systemtimeout`, `/rebroadcastlatencymode` | βœ… Configuration | Device state management | -| **Software Updates** | `/swUpdateCheck`, `/swUpdateQuery`, `/swUpdateAbort`, `/swUpdateStart` | βœ… Update process | Firmware management | -| **Audio Processing** | `/DSPMonoStereo`, `/audiospeakerattributeandsetting` | βœ… Hardware-specific | Advanced audio features | +| Category | Endpoints | Wiki Coverage | Notes | +|----------------------|-------------------------------------------------------------------------------|---------------------|-------------------------| +| **Stereo Pairs** | `/getGroup`, `/addGroup`, `/removeGroup`, `/updateGroup` | βœ… ST-10 specific | L/R speaker pairing | +| **System Info** | `/soundTouchConfigurationStatus`, `/systemtimeout`, `/rebroadcastlatencymode` | βœ… Configuration | Device state management | +| **Software Updates** | `/swUpdateCheck`, `/swUpdateQuery`, `/swUpdateAbort`, `/swUpdateStart` | βœ… Update process | Firmware management | +| **Audio Processing** | `/DSPMonoStereo`, `/audiospeakerattributeandsetting` | βœ… Hardware-specific | Advanced audio features | --- @@ -137,7 +137,7 @@ The SoundTouch Plus community wiki documents **87 distinct API endpoints** with **WebSocket Events Documented:** - `presetsUpdated` - Preset changes -- `groupUpdated` - Stereo pair changes +- `groupUpdated` - Stereo pair changes - `zoneUpdated` - Multi-room changes - `nowPlayingUpdated` - Source/playback changes - `volumeUpdated` - Volume/mute changes @@ -194,7 +194,7 @@ func (c *Client) RateCurrentTrack(rating RatingValue) error func (c *Client) CreateStereoPair(leftIP, rightIP string, name string) error func (c *Client) GetStereoPairStatus() (*StereoPair, error) -// System Management +// System Management func (c *Client) CheckSoftwareUpdate() (*UpdateInfo, error) func (c *Client) GetSystemTimeout() (*TimeoutConfig, error) ``` @@ -283,7 +283,7 @@ The SoundTouch Plus Wiki represents a **treasure trove** of production-ready API ### Key Opportunities: - 🎯 **3x Coverage Expansion**: From 23 to 87+ endpoints - 🏠 **Smart Home Ready**: Complete automation integration -- 🎡 **Music Service Integration**: Full streaming service support +- 🎡 **Music Service Integration**: Full streaming service support - πŸ“± **Professional Features**: Advanced audio and system control - βœ… **Production Ready**: Real-world tested examples and error handling @@ -297,4 +297,4 @@ The SoundTouch Plus Wiki represents a **treasure trove** of production-ready API --- -*Note: All endpoints documented in the wiki are tested against real hardware. Device-specific limitations are clearly documented with compatibility matrices for ST-10, ST-300, and other SoundTouch models.* \ No newline at end of file +*Note: All endpoints documented in the wiki are tested against real hardware. Device-specific limitations are clearly documented with compatibility matrices for ST-10, ST-300, and other SoundTouch models.* diff --git a/docs/concepts/README.md b/docs/concepts/README.md new file mode 100644 index 0000000..51cbd63 --- /dev/null +++ b/docs/concepts/README.md @@ -0,0 +1,181 @@ +# Upstream Bose Service Simulation - Concept Overview + +## Executive Summary + +This document serves as the entry point for understanding the comprehensive plan to enhance the SoundTouch service with advanced state management capabilities, preparing for the eventual shutdown of Bose's upstream services while providing a superior local management experience. + +## Project Objectives + +### Primary Goal +Create a robust, local replacement for Bose's upstream services that can seamlessly handle the transition from cloud-dependent to fully autonomous operation while maintaining and improving upon the existing functionality. + +### Key Outcomes +- **Zero-downtime transition** from Bose services to local management +- **Enhanced visibility** into device states, health, and system operations +- **Data preservation** during migrations with full rollback capabilities +- **Improved reliability** through local control and reduced external dependencies +- **Future-proof architecture** that can evolve beyond Bose's original design + +## Architecture Vision + +### Current State +The existing SoundTouch service provides: +- BMX service for TuneIn integration +- Marge service for account and device management +- Basic mirroring of upstream Bose endpoints +- File-based persistence for device data +- Migration support for device directory structures + +### Enhanced State (This Project) +The enhanced system will add: +- **Comprehensive Account Management** with explicit creation and migration tracking +- **Device Lifecycle Management** with full state machine and event processing +- **Advanced Mirroring** with disparity detection and analysis +- **Dual-Source Data Management** supporting gradual migration strategies +- **Real-time Monitoring** with health checks and performance metrics +- **Text-based Storage** optimized for debugging and small hardware deployments + +## Use Case Coverage + +### Case 0: Account Management +- **Explicit Account Creation**: Accounts created through deliberate user action +- **Mirror-Enhanced Setup**: Use upstream data to enrich account creation +- **Passive Data Collection**: Record account information during normal operations + +### Case 1a: Fresh Device Registration +- **Factory Reset Support**: Handle devices with no prior Bose association +- **Default Configuration**: Initialize devices with sensible presets and sources +- **Local-First Setup**: Complete registration without upstream dependencies + +### Case 1b: Bose Account Migration +- **Data Preservation**: Maintain existing presets, recents, and sources +- **Gradual Migration**: Support partial migration while maintaining upstream compatibility +- **Rollback Capability**: Revert to Bose services if needed + +### Case 2: Lifecycle and State Management +- **Real-time State Tracking**: Monitor device states and health continuously +- **Event-Driven Updates**: Process device events asynchronously +- **Disparity Detection**: Identify differences between local and upstream behavior +- **Comprehensive Logging**: Maintain detailed audit trails for troubleshooting + +## Technical Approach + +### Design Principles +1. **Text-First Storage**: Human-readable formats (JSON, XML, logs) for easy debugging +2. **Small Hardware Optimization**: Designed for Raspberry Pi Zero 2W deployments +3. **Mirror-First Strategy**: Keep upstream mirroring active until migration complete +4. **Event-Driven Architecture**: Asynchronous processing with comprehensive event tracking +5. **Backward Compatibility**: Seamless integration with existing installations + +### Data Structure +``` +data/ +β”œβ”€β”€ accounts/{account-id}/ +β”‚ β”œβ”€β”€ account.json # Account metadata and settings +β”‚ β”œβ”€β”€ account-events.log # High-level account behavior tracking +β”‚ β”œβ”€β”€ devices/{device-id}/ +β”‚ β”‚ β”œβ”€β”€ lifecycle.json # Device state and history +β”‚ β”‚ β”œβ”€β”€ info.xml # Device information (existing) +β”‚ β”‚ β”œβ”€β”€ presets.xml # Device presets (existing) +β”‚ β”‚ β”œβ”€β”€ recents.xml # Recent plays (existing) +β”‚ β”‚ β”œβ”€β”€ sources.xml # Configured sources (existing) +β”‚ β”‚ └── events.log # Device event history +β”‚ └── sessions/ # Recorded interaction sessions (existing) +└── system/ + β”œβ”€β”€ discovery.log # Device discovery events + └── migration.log # Migration activities +``` + +### Development Targets +- **Simplicity**: Keep It Simple, Stupid (KISS) principle over optimization +- **Quality**: 100% test pass rate and lint-clean code for every change +- **Compatibility**: Zero breaking changes to existing functionality +- **Leveraging**: Reuse existing systems (interaction recording, parity detection) + +## Implementation Strategy + +### Phase 1: Foundation (2-3 weeks) - Small, Testable Steps +- Account management foundation with basic create/read operations +- Device lifecycle data models and simple state tracking +- Basic API endpoints with comprehensive testing +- Integration with existing datastore patterns + +### Phase 2: Device Lifecycle (2-3 weeks) - Build on Existing Systems +- Event processing using existing WebSocket system +- Lifecycle integration with current discovery and migration +- Enhanced logging building on existing parity detection +- Simple state machine with thorough testing + +### Phase 3: Enhanced Features (2-3 weeks) - Leverage Current Systems +- Improve existing parity mismatch detection with better categorization +- Smart data source routing with fallback mechanisms +- Basic monitoring using existing health check patterns +- Reuse interaction recording for request/response tracking + +## Key Benefits + +### For Users +- **Continuity**: Seamless operation when Bose services shut down +- **Reliability**: Local control reduces dependency on external services +- **Visibility**: Clear insight into device states and system health +- **Control**: Full management of device data and configurations + +### For Developers +- **Simplicity**: KISS principle makes code easy to understand and maintain +- **Quality**: Comprehensive testing and linting ensures reliable code +- **Debugging**: Text-based storage enables easy troubleshooting +- **Testing**: Every change requires full test suite pass and lint compliance + +### For Community +- **Open Source**: Transparent implementation available for community contributions +- **Standards**: Well-documented APIs and data formats +- **Collaboration**: Disparity detection helps improve implementation accuracy +- **Future-Proof**: Architecture designed to outlast original Bose services + +### Technical Risks +- **Data Loss Prevention**: Atomic file operations and comprehensive testing +- **Complexity Creep**: KISS principle and simple-first approach +- **Compatibility Issues**: Extensive regression testing and existing system reuse +- **Code Quality**: Mandatory linting and test coverage for every change + +### Operational Risks +- **Service Disruption**: Small, incremental changes with rollback capability +- **Testing Overhead**: Automated quality gates (`golangci-lint run --fix` + `go test ./...`) +- **Migration Challenges**: Leverage existing migration system and patterns +- **Maintenance Burden**: Simple, well-tested code is easier to maintain + +### Technical +- All tests pass consistently (100%) +- Zero linting issues in codebase +- No breaking changes to existing functionality +- Code coverage maintained or improved + +### Quality Assurance +- Every commit passes `golangci-lint run --fix` +- Every milestone passes `go test ./...` +- Integration tests verify existing functionality +- Simple, maintainable code that follows Go idioms + +## Documentation Structure + +This concept is detailed across several documents: + +- **[upstream-service-simulation.md](./upstream-service-simulation.md)**: Complete architectural concept with detailed use cases and implementation guidelines +- **[implementation-roadmap.md](./implementation-roadmap.md)**: Detailed project phases, milestones, and delivery timeline +- **[technical-specification.md](./technical-specification.md)**: Comprehensive technical details including APIs, data models, and performance requirements + +## Getting Started + +1. **Review the Concept**: Read through the main concept document to understand the full scope +2. **Examine Technical Details**: Review the technical specification for implementation details +3. **Follow the Roadmap**: Use the implementation roadmap for project planning and execution +4. **Integration Planning**: Consider how the enhanced features will integrate with existing deployments + +## Next Steps + +1. **Stakeholder Review**: Gather feedback on the concept and approach +2. **Technical Validation**: Prototype key components to validate technical assumptions +3. **Resource Planning**: Allocate development resources for the three-phase implementation +4. **Community Engagement**: Share plans with the community for feedback and contributions + +This enhanced state management system represents a significant evolution of the SoundTouch service, transforming it from a basic cloud replacement into a comprehensive, future-proof device management platform that can serve users well beyond the Bose service shutdown timeline. \ No newline at end of file diff --git a/docs/concepts/implementation-plan.md b/docs/concepts/implementation-plan.md new file mode 100644 index 0000000..c49cbc8 --- /dev/null +++ b/docs/concepts/implementation-plan.md @@ -0,0 +1,355 @@ +# Implementation Plan - Enhanced State Management System + +## Overview + +This document provides a detailed, step-by-step implementation plan for the enhanced state management system. Each step is designed to be small, testable, and independently valuable while maintaining backward compatibility. + +## Development Principles + +### Quality Gates +Every step must pass these checks before proceeding: +1. `golangci-lint run --fix` - no linting issues +2. `go test ./...` - all tests pass +3. Existing functionality remains intact +4. New functionality has appropriate test coverage + +### KISS Principle +- Write the simplest code that works +- Avoid premature optimization +- Use straightforward algorithms +- Build incrementally with small changes + +### Leverage Existing Systems +- Reuse interaction recording for request/response tracking +- Build upon current parity mismatch detection +- Extend existing datastore patterns +- Integrate with established workflows + +## Phase 1: Foundation Preparation (2-3 weeks) + +### Step 1.1: Code Organization Preparation +**Duration**: 2-3 days +**Goal**: Prepare package structure without changing behavior + +#### Mini-milestone 1.1.1: Create account package structure +- Create `pkg/service/account/` directory +- Add basic `account.go` with placeholder structs +- Add `account_test.go` with basic test structure +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 1.1.2: Create lifecycle package structure +- Create `pkg/service/lifecycle/` directory +- Add basic `lifecycle.go` with placeholder structs +- Add `lifecycle_test.go` with basic test structure +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 1.1.3: Extend datastore interface preparation +- Add placeholder methods to existing datastore for account operations +- Ensure all existing functionality still works +- Add tests for new placeholder methods +- **Quality Check**: Lint + test all packages + +### Step 1.2: Account Management Foundation +**Duration**: 3-4 days +**Goal**: Basic account creation and retrieval + +#### Mini-milestone 1.2.1: Account data model +- Define `Account` struct with basic fields +- Add validation functions +- Add comprehensive unit tests +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 1.2.2: Account persistence +- Implement account.json file read/write +- Add atomic file operations +- Test file operations thoroughly +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 1.2.3: Account manager basic operations +- Implement `CreateAccount()` function +- Implement `GetAccount()` function +- Add error handling and validation +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 1.2.4: Integration with existing datastore +- Modify datastore to use account manager +- Ensure backward compatibility with existing accounts +- Test migration of existing data structure +- **Quality Check**: Lint + test all packages + +### Step 1.3: Basic API Endpoints +**Duration**: 2-3 days +**Goal**: Add REST endpoints for account management + +#### Mini-milestone 1.3.1: Account creation endpoint +- Add `POST /api/v1/accounts` handler +- Integrate with existing HTTP router +- Add input validation and error responses +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 1.3.2: Account retrieval endpoint +- Add `GET /api/v1/accounts/{id}` handler +- Add proper JSON serialization +- Test endpoint functionality +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 1.3.3: Integration testing +- Test new endpoints with existing functionality +- Ensure XML endpoints still work +- Verify no breaking changes +- **Quality Check**: Lint + test all packages + +## Phase 2: Device Lifecycle Foundation (2-3 weeks) + +### Step 2.1: Device State Model +**Duration**: 3-4 days +**Goal**: Basic device lifecycle tracking + +#### Mini-milestone 2.1.1: Device lifecycle data model +- Define `DeviceLifecycle` struct +- Define device states and transitions +- Add validation and helper functions +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 2.1.2: State transition logic +- Implement basic state machine +- Add transition validation +- Create comprehensive tests for all transitions +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 2.1.3: Lifecycle persistence +- Implement lifecycle.json file operations +- Add atomic updates and error handling +- Test persistence thoroughly +- **Quality Check**: Lint + test all packages + +### Step 2.2: Event Processing Foundation +**Duration**: 3-4 days +**Goal**: Basic event handling and logging + +#### Mini-milestone 2.2.1: Event data model +- Define `DeviceEvent` struct +- Add event types and validation +- Create event builder helpers +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 2.2.2: Simple event logging +- Implement append-only event log writing +- Add structured log format +- Test log operations and rotation +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 2.2.3: Event processing pipeline +- Create basic synchronous event processor +- Add event validation and filtering +- Integrate with existing WebSocket events +- **Quality Check**: Lint + test all packages + +### Step 2.3: Lifecycle Integration +**Duration**: 2-3 days +**Goal**: Connect lifecycle to existing systems + +#### Mini-milestone 2.3.1: Discovery integration +- Trigger lifecycle events on device discovery +- Update device state on discovery +- Test discovery workflow with lifecycle +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 2.3.2: WebSocket integration +- Process WebSocket events through lifecycle +- Update device state based on events +- Log significant state changes +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 2.3.3: Migration integration +- Integrate lifecycle with existing migration system +- Track migration events and state changes +- Ensure existing migration still works +- **Quality Check**: Lint + test all packages + +## Phase 3: Enhanced Features (2-3 weeks) + +### Step 3.1: Enhanced Mirroring +**Duration**: 3-4 days +**Goal**: Improve existing parity detection + +#### Mini-milestone 3.1.1: Extended disparity logging +- Enhance existing parity mismatch logging +- Add more detailed disparity information +- Improve log format for analysis +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 3.1.2: Disparity categorization +- Add severity levels to disparities +- Categorize different types of mismatches +- Add filtering and search capabilities +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 3.1.3: Enhanced mirror middleware +- Extend existing mirror functionality +- Add better response comparison +- Integrate with lifecycle events +- **Quality Check**: Lint + test all packages + +### Step 3.2: Data Source Management +**Duration**: 3-4 days +**Goal**: Smart routing between local and upstream + +#### Mini-milestone 3.2.1: Data source configuration +- Add per-device source preferences +- Implement source switching logic +- Add configuration persistence +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 3.2.2: Fallback mechanisms +- Add graceful fallback on source failure +- Implement simple health checking +- Test fallback scenarios +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 3.2.3: Migration orchestration +- Add device-by-device migration control +- Track migration progress +- Add rollback capabilities +- **Quality Check**: Lint + test all packages + +### Step 3.3: Monitoring and Health +**Duration**: 2-3 days +**Goal**: Basic system monitoring + +#### Mini-milestone 3.3.1: Health check endpoints +- Add system health endpoints +- Report service status +- Add basic metrics collection +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 3.3.2: Device health tracking +- Track device connectivity +- Monitor response times +- Log health status changes +- **Quality Check**: Lint + test all packages + +#### Mini-milestone 3.3.3: System metrics +- Add basic performance metrics +- Track resource usage +- Add metrics endpoints +- **Quality Check**: Lint + test all packages + +## Quality Assurance Strategy + +### Testing Requirements +Each mini-milestone must include: +- Unit tests for new functions +- Integration tests for modified workflows +- Regression tests for existing functionality +- Performance tests for critical paths + +### Test Categories + +#### Unit Tests +- Test individual functions and methods +- Mock external dependencies +- Cover error conditions and edge cases +- Aim for >90% code coverage on new code + +#### Integration Tests +- Test component interactions +- Use real file operations in test environment +- Test HTTP endpoints end-to-end +- Verify existing functionality unchanged + +#### Regression Tests +- Ensure existing XML endpoints work +- Verify device discovery still functions +- Check migration compatibility +- Test WebSocket event processing + +### Continuous Quality Checks + +#### Pre-commit Checks +```bash +# Before each commit +golangci-lint run --fix +go test ./... +go test -race ./... +``` + +#### Milestone Validation +```bash +# Before marking milestone complete +golangci-lint run --fix +go test ./... -v +go test -race ./... -v +go test ./... -bench=. +``` + +#### Integration Validation +```bash +# Test with real soundtouch-service +make build +./soundtouch-service & +# Run integration test suite +make integration-test +``` + +## Risk Mitigation + +### Backward Compatibility +- All existing APIs must continue working +- File structure changes must be additive +- Configuration changes must have defaults +- Migration paths for existing data + +### Rollback Strategy +- Each step can be independently reverted +- Configuration flags for new features +- Graceful degradation when features disabled +- Clear rollback documentation + +### Performance Impact +- Monitor memory usage during development +- Profile critical paths before and after changes +- Set performance regression alerts +- Simple before complex solutions + +## Documentation Requirements + +### Code Documentation +- Comprehensive godoc comments +- Example usage in comments +- Error conditions documented +- Performance characteristics noted + +### User Documentation +- Update existing guides for new features +- Add migration guides for new functionality +- Create troubleshooting documentation +- Update API documentation + +### Development Documentation +- Architecture decision records +- Testing strategy documentation +- Deployment and rollback procedures +- Performance benchmarking results + +## Success Criteria + +### Technical Metrics +- All tests pass consistently +- No linting issues +- Memory usage increase <50MB +- Response time degradation <10% + +### Functional Metrics +- All existing functionality preserved +- New account management works reliably +- Device lifecycle tracking is accurate +- Enhanced monitoring provides value + +### Quality Metrics +- Code coverage maintained >85% +- No critical security issues +- Documentation completeness >95% +- Community feedback positive + +This implementation plan ensures steady, reliable progress while maintaining the quality and simplicity principles essential for the project's success. \ No newline at end of file diff --git a/docs/concepts/implementation-roadmap.md b/docs/concepts/implementation-roadmap.md new file mode 100644 index 0000000..87d09df --- /dev/null +++ b/docs/concepts/implementation-roadmap.md @@ -0,0 +1,403 @@ +# Implementation Roadmap for Upstream Service Simulation + +## Overview + +This document provides a detailed implementation roadmap for the upstream Bose service simulation concept. It breaks down the implementation into manageable phases with specific deliverables, technical requirements, and integration points. + +## Phase 1: Foundation and Enhanced State Tracking (4-6 weeks) + +### Milestone 1.1: Account Management Service (1-2 weeks) + +#### Deliverables +- `pkg/service/account/` package with core account management +- Account creation, retrieval, and status management APIs +- Text-based account persistence in JSON format +- Integration with existing datastore structure + +#### Implementation Tasks +1. **Create Account Manager** + ``` + pkg/service/account/ + β”œβ”€β”€ account.go # Core account management + β”œβ”€β”€ manager.go # Account manager implementation + β”œβ”€β”€ persistence.go # File-based persistence + └── account_test.go # Comprehensive tests + ``` + +2. **Account Data Structure** + - JSON-based account metadata storage + - Integration with existing `data/accounts/{id}/` structure + - Account status tracking (active, migrating, suspended) + - Migration metadata tracking + +3. **API Integration** + - Add account management endpoints to existing HTTP router + - RESTful API alongside existing XML endpoints + - Account creation validation and error handling + +#### Technical Requirements +- Maintain backward compatibility with existing account structure +- Thread-safe account operations +- Atomic file operations for account metadata +- Comprehensive error handling and logging + +### Milestone 1.2: Device Lifecycle Manager (2-3 weeks) + +#### Deliverables +- `pkg/service/lifecycle/` package for device state management +- Device state machine with comprehensive state tracking +- Event-driven state transitions +- Integration with existing device discovery and migration + +#### Implementation Tasks +1. **Lifecycle Core** + ``` + pkg/service/lifecycle/ + β”œβ”€β”€ lifecycle.go # Device lifecycle management + β”œβ”€β”€ states.go # State definitions and transitions + β”œβ”€β”€ events.go # Event processing + β”œβ”€β”€ persistence.go # Lifecycle persistence + └── lifecycle_test.go # State machine tests + ``` + +2. **State Machine Implementation** + - Define device states: unregistered β†’ registering β†’ active β†’ migrating β†’ offline β†’ retired + - Implement state transition rules and validation + - Event-driven state changes with history tracking + - Integration with existing migration system + +3. **Event Processing** + - Asynchronous event queue for device events + - Event categorization and filtering + - Text-based event logging with structured format + - Event replay capabilities for debugging + +#### Technical Requirements +- Non-blocking event processing +- Persistent state across service restarts +- Integration with existing WebSocket event system +- Memory-efficient event storage + +### Milestone 1.3: Enhanced Mirror System (1-2 weeks) + +#### Deliverables +- Extended mirroring with disparity detection +- Parity analysis logging and reporting +- Selective data source switching +- Integration with existing mirror middleware + +#### Implementation Tasks +1. **Disparity Detection** + ``` + pkg/service/mirror/ + β”œβ”€β”€ disparity.go # Disparity detection logic + β”œβ”€β”€ analyzer.go # Response analysis and comparison + β”œβ”€β”€ logger.go # Structured disparity logging + └── disparity_test.go # Analysis tests + ``` + +2. **Enhanced Mirror Middleware** + - Extend existing mirror functionality + - Add response comparison and hash calculation + - Structured logging of disparities + - Configurable disparity sensitivity + +3. **Data Source Management** + - Smart routing between local and upstream sources + - Per-endpoint source preference configuration + - Fallback mechanisms for upstream unavailability + - Source switching with history tracking + +#### Technical Requirements +- Minimal performance impact on request processing +- Configurable disparity detection sensitivity +- Structured logging for analysis tools +- Integration with existing mirror configuration + +## Phase 2: Migration and Dual-Source Management (3-4 weeks) + +### Milestone 2.1: Migration Controller (2-3 weeks) + +#### Deliverables +- Device-by-device migration orchestration +- Migration progress tracking and status reporting +- Rollback capabilities with state preservation +- Integration with existing setup manager + +#### Implementation Tasks +1. **Migration Orchestration** + ``` + pkg/service/migration/ + β”œβ”€β”€ controller.go # Migration orchestration + β”œβ”€β”€ strategy.go # Migration strategies + β”œβ”€β”€ rollback.go # Rollback functionality + β”œβ”€β”€ progress.go # Progress tracking + └── migration_integration_test.go + ``` + +2. **Migration Strategies** + - Fresh device registration flow + - Bose account data migration flow + - Gradual migration with dual-source support + - Emergency migration for service outages + +3. **Progress Tracking** + - Real-time migration status updates + - Migration timeline and milestone tracking + - Error handling and recovery procedures + - Migration completion verification + +#### Technical Requirements +- Integration with existing migration system +- Atomic migration operations with rollback +- Progress persistence across service restarts +- Comprehensive migration logging + +### Milestone 2.2: Dual-Source Data Management (1-2 weeks) + +#### Deliverables +- Smart data routing between local and upstream sources +- Graceful fallback mechanisms +- Data source preference management +- Conflict resolution strategies + +#### Implementation Tasks +1. **Data Source Router** + ``` + pkg/service/datasource/ + β”œβ”€β”€ router.go # Smart routing logic + β”œβ”€β”€ preferences.go # Source preference management + β”œβ”€β”€ fallback.go # Fallback mechanisms + └── conflict.go # Conflict resolution + ``` + +2. **Source Management** + - Per-device, per-endpoint source preferences + - Dynamic source switching based on availability + - Conflict detection and resolution + - Source health monitoring + +3. **Integration Points** + - Marge service integration for account data + - BMX service integration for content data + - Preset and recent management integration + - Source configuration management + +#### Technical Requirements +- Zero-downtime source switching +- Conflict resolution without data loss +- Health check integration +- Performance monitoring and metrics + +## Phase 3: Advanced Features and Analytics (2-3 weeks) + +### Milestone 3.1: System Monitoring and Health Checks (1-2 weeks) + +#### Deliverables +- Comprehensive system health monitoring +- Device connectivity and availability tracking +- Performance metrics collection +- Health check endpoints and dashboards + +#### Implementation Tasks +1. **Health Monitoring** + ``` + pkg/service/health/ + β”œβ”€β”€ monitor.go # System health monitoring + β”œβ”€β”€ metrics.go # Performance metrics + β”œβ”€β”€ connectivity.go # Device connectivity tracking + └── alerts.go # Health alerting + ``` + +2. **Metrics Collection** + - Device availability tracking + - Response time monitoring + - Error rate tracking + - Migration success rates + +3. **Dashboard Integration** + - Health status endpoints + - Metrics export for monitoring tools + - Real-time status updates + - Historical trend analysis + +#### Technical Requirements +- Minimal performance overhead +- Configurable monitoring intervals +- Integration with existing health checks +- Memory-efficient metrics storage + +### Milestone 3.2: Data Export and Backup (1 week) + +#### Deliverables +- Account data export functionality +- Incremental backup strategies +- Data integrity verification +- Migration-ready data formats + +#### Implementation Tasks +1. **Export Functionality** + ``` + pkg/service/export/ + β”œβ”€β”€ exporter.go # Data export logic + β”œβ”€β”€ formats.go # Export format definitions + β”œβ”€β”€ validation.go # Data integrity checks + └── backup.go # Backup strategies + ``` + +2. **Backup Management** + - Incremental backup creation + - Backup validation and verification + - Automated backup scheduling + - Restore functionality + +3. **Data Formats** + - Migration-ready JSON exports + - XML compatibility for device imports + - Compressed archive support + - Selective export capabilities + +#### Technical Requirements +- Consistent data export across all account types +- Backup integrity verification +- Configurable export scheduling +- Resource-efficient backup operations + +## Integration Strategy + +### Existing Service Integration Points + +#### 1. Datastore Integration +- Extend existing datastore with lifecycle and account management +- Maintain backward compatibility with current file structure +- Add new persistence methods for enhanced state tracking +- Implement migration for existing data to new formats + +#### 2. Handler Integration +- Integrate account management into existing HTTP handlers +- Add lifecycle information to device responses +- Extend mirror middleware with disparity detection +- Add new management endpoints alongside existing XML APIs + +#### 3. Discovery Integration +- Link device discovery to lifecycle state transitions +- Integrate migration triggers with discovery events +- Add account association during discovery +- Maintain existing discovery functionality + +#### 4. Migration System Integration +- Extend existing migration manager with new capabilities +- Integrate lifecycle management with device migrations +- Add rollback functionality to existing migration flows +- Maintain compatibility with current migration methods + +### Configuration Management + +#### New Configuration Options +```yaml +accounts: + auto_create: false + mirror_enhanced_creation: true + default_migration_strategy: "gradual" + +lifecycle: + event_retention_days: 30 + state_transition_timeout: "5m" + async_processing: true + +mirror: + disparity_detection: true + disparity_sensitivity: "medium" + source_switching_enabled: true + fallback_timeout: "10s" + +migration: + batch_size: 1 + progress_reporting: true + rollback_enabled: true + verification_required: true +``` + +### Performance Considerations + +#### Resource Usage +- Target: <100MB additional memory usage on Raspberry Pi Zero 2W +- CPU usage: <5% additional overhead during normal operations +- Storage: Text-based logs with configurable rotation +- Network: Minimal additional upstream requests + +#### Optimization Strategies +- Lazy loading of historical data +- Configurable log retention policies +- Memory-efficient event processing +- Background cleanup processes +- Efficient file I/O operations + +## Testing Strategy + +### Unit Testing +- Comprehensive test coverage for all new packages +- State machine transition testing +- Data persistence and integrity tests +- Mock integration tests for external dependencies + +### Integration Testing +- End-to-end migration flow testing +- Multi-device scenario testing +- Disparity detection accuracy testing +- Performance impact testing + +### Compatibility Testing +- Backward compatibility with existing installations +- Device compatibility across SoundTouch models +- Migration from various existing configurations +- Stress testing with multiple concurrent devices + +## Deployment Strategy + +### Rollout Plan +1. **Alpha Release**: Core functionality with limited device support +2. **Beta Release**: Full feature set with extensive testing +3. **Stable Release**: Production-ready with documentation + +### Migration Path +1. Existing installations can upgrade incrementally +2. New features are opt-in with configuration flags +3. Existing data structures are preserved and extended +4. Rollback capability for critical issues + +### Documentation Requirements +- Updated API documentation with new endpoints +- Migration guide for existing users +- Configuration reference for new options +- Troubleshooting guide for common issues + +## Risk Mitigation + +### Technical Risks +- **Data Loss**: Atomic operations and rollback capabilities +- **Performance Impact**: Gradual rollout and monitoring +- **Compatibility Issues**: Comprehensive testing and fallback options +- **Resource Constraints**: Efficient algorithms and configurable limits + +### Operational Risks +- **Service Disruption**: Zero-downtime deployment strategies +- **Configuration Complexity**: Sensible defaults and validation +- **User Adoption**: Clear documentation and migration assistance +- **Support Burden**: Comprehensive logging and diagnostic tools + +## Success Metrics + +### Technical Metrics +- Migration success rate >95% +- Disparity detection accuracy >90% +- Performance overhead <5% +- System availability >99.5% + +### User Experience Metrics +- Reduced support requests +- Improved device reliability +- Faster problem resolution +- Enhanced system visibility + +This roadmap provides a structured approach to implementing the upstream service simulation concept while maintaining compatibility with existing deployments and ensuring smooth migration paths for users. \ No newline at end of file diff --git a/docs/concepts/technical-specification.md b/docs/concepts/technical-specification.md new file mode 100644 index 0000000..9ea2f73 --- /dev/null +++ b/docs/concepts/technical-specification.md @@ -0,0 +1,989 @@ +# Technical Specification - Enhanced State Management System + +## Table of Contents + +1. [System Architecture](#system-architecture) +2. [Data Models](#data-models) +3. [API Specifications](#api-specifications) +4. [File Format Specifications](#file-format-specifications) +5. [State Machine Definitions](#state-machine-definitions) +6. [Event Processing](#event-processing) +7. [Performance Requirements](#performance-requirements) +8. [Security Considerations](#security-considerations) +9. [Error Handling](#error-handling) +10. [Monitoring and Observability](#monitoring-and-observability) + +## System Architecture + +### Component Overview + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ SoundTouch Service β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ HTTP Router & Middleware β”‚ +β”‚ β”œβ”€β”€ Mirror Middleware (Enhanced) β”‚ +β”‚ β”œβ”€β”€ Recorder Middleware β”‚ +β”‚ β”œβ”€β”€ Disparity Detection β”‚ +β”‚ └── Health Check Middleware β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Service Layer β”‚ +β”‚ β”œβ”€β”€ Account Manager β”œβ”€β”€ Lifecycle Manager β”‚ +β”‚ β”œβ”€β”€ Migration Controller β”œβ”€β”€ Data Source Router β”‚ +β”‚ β”œβ”€β”€ Event Processor β”œβ”€β”€ Health Monitor β”‚ +β”‚ └── Export Manager └── Analytics Engine β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Data Layer β”‚ +β”‚ β”œβ”€β”€ Enhanced DataStore β”œβ”€β”€ Event Store β”‚ +β”‚ β”œβ”€β”€ Mirror Cache β”œβ”€β”€ Metrics Store β”‚ +β”‚ └── Configuration Store └── Session Store β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ External Integrations β”‚ +β”‚ β”œβ”€β”€ Bose Services (Mirror) β”œβ”€β”€ Device Discovery β”‚ +β”‚ β”œβ”€β”€ BMX/TuneIn Services └── SSH/Setup Manager β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### Package Structure + +``` +pkg/service/ +β”œβ”€β”€ account/ # Account management +β”‚ β”œβ”€β”€ manager.go +β”‚ β”œβ”€β”€ persistence.go +β”‚ └── validation.go +β”œβ”€β”€ lifecycle/ # Device lifecycle management +β”‚ β”œβ”€β”€ manager.go +β”‚ β”œβ”€β”€ states.go +β”‚ β”œβ”€β”€ transitions.go +β”‚ └── events.go +β”œβ”€β”€ migration/ # Enhanced migration (extends existing) +β”‚ β”œβ”€β”€ controller.go +β”‚ β”œβ”€β”€ strategies.go +β”‚ └── progress.go +β”œβ”€β”€ datasource/ # Data source routing +β”‚ β”œβ”€β”€ router.go +β”‚ β”œβ”€β”€ preferences.go +β”‚ └── fallback.go +β”œβ”€β”€ events/ # Event processing system +β”‚ β”œβ”€β”€ processor.go +β”‚ β”œβ”€β”€ queue.go +β”‚ └── storage.go +β”œβ”€β”€ mirror/ # Enhanced mirroring (extends existing) +β”‚ β”œβ”€β”€ disparity.go +β”‚ β”œβ”€β”€ analyzer.go +β”‚ └── logger.go +β”œβ”€β”€ health/ # System monitoring +β”‚ β”œβ”€β”€ monitor.go +β”‚ β”œβ”€β”€ metrics.go +β”‚ └── alerts.go +└── export/ # Data export and backup + β”œβ”€β”€ exporter.go + β”œβ”€β”€ formats.go + └── backup.go +``` + +## Data Models + +### Account Model + +```go +type Account struct { + ID string `json:"id"` + Name string `json:"name"` + Email string `json:"email,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Status AccountStatus `json:"status"` + DeviceCount int `json:"device_count"` + MigrationInfo *MigrationInfo `json:"migration_info,omitempty"` + BoseAccountID string `json:"bose_account_id,omitempty"` + DataSources DataSourceConfig `json:"data_sources"` + Settings AccountSettings `json:"settings"` +} + +type AccountStatus string +const ( + AccountStatusActive AccountStatus = "active" + AccountStatusMigrating AccountStatus = "migrating" + AccountStatusSuspended AccountStatus = "suspended" + AccountStatusArchived AccountStatus = "archived" +) + +type MigrationInfo struct { + StartedAt time.Time `json:"started_at"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + DevicesMigrated int `json:"devices_migrated"` + DevicesPending int `json:"devices_pending"` + MirrorActive bool `json:"mirror_active"` + Strategy string `json:"strategy"` + RollbackData string `json:"rollback_data,omitempty"` +} + +type DataSourceConfig struct { + Local bool `json:"local"` + BoseMirror bool `json:"bose_mirror"` + Primary string `json:"primary"` // "local" or "bose" +} + +type AccountSettings struct { + AutoMigration bool `json:"auto_migration"` + MirrorEndpoints []string `json:"mirror_endpoints"` + RetentionDays int `json:"retention_days"` +} +``` + +### Device Lifecycle Model + +```go +type DeviceLifecycle struct { + DeviceID string `json:"device_id"` + AccountID string `json:"account_id"` + State DeviceState `json:"state"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + StateHistory []StateTransition `json:"state_history"` + Metadata DeviceMetadata `json:"metadata"` + DataSources DataSourceConfig `json:"data_sources"` + Migration *DeviceMigration `json:"migration,omitempty"` + Health DeviceHealth `json:"health"` +} + +type DeviceState string +const ( + DeviceStateUnregistered DeviceState = "unregistered" + DeviceStateDiscovered DeviceState = "discovered" + DeviceStateRegistering DeviceState = "registering" + DeviceStateActive DeviceState = "active" + DeviceStateMigrating DeviceState = "migrating" + DeviceStateOffline DeviceState = "offline" + DeviceStateError DeviceState = "error" + DeviceStateRetired DeviceState = "retired" +) + +type StateTransition struct { + From DeviceState `json:"from"` + To DeviceState `json:"to"` + Timestamp time.Time `json:"timestamp"` + Reason string `json:"reason"` + Source string `json:"source"` + Context map[string]interface{} `json:"context,omitempty"` +} + +type DeviceMetadata struct { + Name string `json:"name"` + Type string `json:"type"` + SerialNumber string `json:"serial_number"` + FirmwareVersion string `json:"firmware_version"` + MACAddress string `json:"mac_address"` + IPAddress string `json:"ip_address"` + LastSeen time.Time `json:"last_seen"` + IsLegacyID bool `json:"is_legacy_id"` + Capabilities []string `json:"capabilities,omitempty"` +} + +type DeviceMigration struct { + FromBoseAccount string `json:"from_bose_account,omitempty"` + MigratedAt *time.Time `json:"migrated_at,omitempty"` + Method string `json:"method"` + RollbackAvailable bool `json:"rollback_available"` + DataPreserved []string `json:"data_preserved"` +} + +type DeviceHealth struct { + Status string `json:"status"` // "healthy", "warning", "error" + LastCheck time.Time `json:"last_check"` + ResponseTime int `json:"response_time_ms"` + Connectivity string `json:"connectivity"` // "online", "offline", "intermittent" + ErrorCount int `json:"error_count"` + LastError string `json:"last_error,omitempty"` +} +``` + +### Event Model + +```go +type DeviceEvent struct { + ID string `json:"id"` + DeviceID string `json:"device_id"` + AccountID string `json:"account_id"` + Type DeviceEventType `json:"type"` + Data map[string]interface{} `json:"data"` + Timestamp time.Time `json:"timestamp"` + Source EventSource `json:"source"` + Processed bool `json:"processed"` + Context EventContext `json:"context"` +} + +type DeviceEventType string +const ( + EventTypeNowPlaying DeviceEventType = "now_playing" + EventTypePresetChanged DeviceEventType = "preset_changed" + EventTypeVolumeChanged DeviceEventType = "volume_changed" + EventTypeSourceChanged DeviceEventType = "source_changed" + EventTypeDeviceOnline DeviceEventType = "device_online" + EventTypeDeviceOffline DeviceEventType = "device_offline" + EventTypeZoneChanged DeviceEventType = "zone_changed" + EventTypeDisparityFound DeviceEventType = "disparity_found" + EventTypeMigrationStart DeviceEventType = "migration_start" + EventTypeMigrationEnd DeviceEventType = "migration_end" + EventTypeHealthCheck DeviceEventType = "health_check" + EventTypeErrorOccurred DeviceEventType = "error_occurred" +) + +type EventSource string +const ( + EventSourceWebSocket EventSource = "websocket" + EventSourceDiscovery EventSource = "discovery" + EventSourceMirror EventSource = "mirror" + EventSourceSystem EventSource = "system" + EventSourceAPI EventSource = "api" + EventSourceUser EventSource = "user" +) + +type EventContext struct { + RequestID string `json:"request_id,omitempty"` + UserAgent string `json:"user_agent,omitempty"` + IPAddress string `json:"ip_address,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Additional map[string]interface{} `json:"additional,omitempty"` +} +``` + +### Disparity Model + +```go +type Disparity struct { + ID string `json:"id"` + Timestamp time.Time `json:"timestamp"` + DeviceID string `json:"device_id"` + AccountID string `json:"account_id"` + Endpoint string `json:"endpoint"` + Type DisparityType `json:"type"` + Severity DisparitySeverity `json:"severity"` + LocalHash string `json:"local_hash"` + UpstreamHash string `json:"upstream_hash"` + Details DisparityDetails `json:"details"` + Context map[string]interface{} `json:"context"` + Resolved bool `json:"resolved"` +} + +type DisparityType string +const ( + DisparityTypeContentMismatch DisparityType = "content_mismatch" + DisparityTypeStructureDiff DisparityType = "structure_diff" + DisparityTypeTimestampFormat DisparityType = "timestamp_format" + DisparityTypeFieldMissing DisparityType = "field_missing" + DisparityTypeValueMismatch DisparityType = "value_mismatch" + DisparityTypeCountMismatch DisparityType = "count_mismatch" +) + +type DisparitySeverity string +const ( + DisparitySeverityLow DisparitySeverity = "low" + DisparitySeverityMedium DisparitySeverity = "medium" + DisparitySeverityHigh DisparitySeverity = "high" + DisparitySeverityCritical DisparitySeverity = "critical" +) + +type DisparityDetails struct { + FieldPath string `json:"field_path"` + LocalValue interface{} `json:"local_value"` + UpstreamValue interface{} `json:"upstream_value"` + Description string `json:"description"` +} +``` + +## API Specifications + +### Account Management APIs + +#### Create Account +```http +POST /api/v1/accounts +Content-Type: application/json + +{ + "name": "User Account", + "email": "user@example.com", + "settings": { + "auto_migration": false, + "retention_days": 30 + } +} + +Response: 201 Created +{ + "id": "acc_12345", + "name": "User Account", + "email": "user@example.com", + "created_at": "2024-01-20T10:00:00Z", + "status": "active", + "device_count": 0 +} +``` + +#### Get Account +```http +GET /api/v1/accounts/{account_id} + +Response: 200 OK +{ + "id": "acc_12345", + "name": "User Account", + "status": "active", + "device_count": 2, + "migration_info": { + "started_at": "2024-01-18T09:00:00Z", + "devices_migrated": 1, + "devices_pending": 1, + "mirror_active": true + }, + "data_sources": { + "local": true, + "bose_mirror": true, + "primary": "bose" + } +} +``` + +#### List Accounts +```http +GET /api/v1/accounts?status=active&limit=10&offset=0 + +Response: 200 OK +{ + "accounts": [...], + "total": 5, + "limit": 10, + "offset": 0 +} +``` + +### Device Lifecycle APIs + +#### Register Device +```http +POST /api/v1/accounts/{account_id}/devices +Content-Type: application/json + +{ + "device_id": "A81B6A536A98", + "name": "Living Room Speaker", + "registration_type": "fresh" +} + +Response: 201 Created +{ + "device_id": "A81B6A536A98", + "account_id": "acc_12345", + "state": "registering", + "created_at": "2024-01-20T10:00:00Z" +} +``` + +#### Get Device State +```http +GET /api/v1/accounts/{account_id}/devices/{device_id}/state + +Response: 200 OK +{ + "device_id": "A81B6A536A98", + "account_id": "acc_12345", + "state": "active", + "metadata": { + "name": "Living Room Speaker", + "type": "SoundTouch 30", + "last_seen": "2024-01-20T15:30:00Z" + }, + "health": { + "status": "healthy", + "connectivity": "online", + "response_time": 45 + } +} +``` + +#### Migrate Device +```http +POST /api/v1/accounts/{account_id}/devices/{device_id}/migrate +Content-Type: application/json + +{ + "from_bose_account": "bose-acc-xyz", + "preserve_data": true, + "method": "gradual" +} + +Response: 202 Accepted +{ + "migration_id": "mig_67890", + "status": "started", + "estimated_completion": "2024-01-20T11:00:00Z" +} +``` + +### Event APIs + +#### Get Device Events +```http +GET /api/v1/accounts/{account_id}/devices/{device_id}/events?since=2024-01-20T00:00:00Z&type=now_playing&limit=50 + +Response: 200 OK +{ + "events": [ + { + "id": "evt_12345", + "type": "now_playing", + "timestamp": "2024-01-20T15:30:00Z", + "data": { + "source": "SPOTIFY", + "track": "Song Name", + "artist": "Artist Name" + } + } + ], + "total": 125, + "has_more": true +} +``` + +#### Stream Events +```http +GET /api/v1/accounts/{account_id}/devices/{device_id}/events/stream +Accept: text/event-stream + +Response: 200 OK +Content-Type: text/event-stream + +data: {"id":"evt_12346","type":"volume_changed","timestamp":"2024-01-20T15:31:00Z","data":{"volume":50}} + +data: {"id":"evt_12347","type":"now_playing","timestamp":"2024-01-20T15:32:00Z","data":{"source":"TUNEIN"}} +``` + +### Monitoring APIs + +#### System Health +```http +GET /api/v1/system/health + +Response: 200 OK +{ + "status": "healthy", + "timestamp": "2024-01-20T15:30:00Z", + "services": { + "account_manager": "healthy", + "lifecycle_manager": "healthy", + "event_processor": "healthy", + "mirror_service": "warning" + }, + "statistics": { + "total_accounts": 5, + "total_devices": 12, + "active_devices": 10, + "events_processed_24h": 1547 + } +} +``` + +#### Disparity Analysis +```http +GET /api/v1/system/disparities?since=2024-01-20T00:00:00Z&severity=high + +Response: 200 OK +{ + "disparities": [ + { + "id": "disp_12345", + "timestamp": "2024-01-20T14:30:00Z", + "endpoint": "/v1/presets", + "type": "count_mismatch", + "severity": "high", + "details": { + "field_path": "preset_count", + "local_value": 5, + "upstream_value": 4 + } + } + ], + "summary": { + "total": 15, + "by_severity": { + "high": 2, + "medium": 8, + "low": 5 + } + } +} +``` + +## File Format Specifications + +### Account Metadata (account.json) +```json +{ + "version": "1.0", + "id": "acc_12345", + "name": "User Account", + "email": "user@example.com", + "created_at": "2024-01-20T10:00:00Z", + "updated_at": "2024-01-20T15:30:00Z", + "status": "active", + "device_count": 2, + "migration_info": { + "started_at": "2024-01-18T09:00:00Z", + "devices_migrated": 1, + "devices_pending": 1, + "mirror_active": true, + "strategy": "gradual" + }, + "bose_account_id": "bose-original-id", + "data_sources": { + "local": true, + "bose_mirror": true, + "primary": "bose" + }, + "settings": { + "auto_migration": false, + "mirror_endpoints": ["/v1/presets", "/v1/recents"], + "retention_days": 30 + } +} +``` + +### Device Lifecycle (lifecycle.json) +```json +{ + "version": "1.0", + "device_id": "A81B6A536A98", + "account_id": "acc_12345", + "state": "active", + "created_at": "2024-01-20T10:00:00Z", + "updated_at": "2024-01-20T15:30:00Z", + "state_history": [ + { + "from": "unregistered", + "to": "discovered", + "timestamp": "2024-01-20T10:00:00Z", + "reason": "mdns_discovery", + "source": "discovery", + "context": { + "ip_address": "192.168.1.100", + "discovery_method": "mdns" + } + }, + { + "from": "discovered", + "to": "active", + "timestamp": "2024-01-20T10:05:00Z", + "reason": "registration_complete", + "source": "system" + } + ], + "metadata": { + "name": "Living Room Speaker", + "type": "SoundTouch 30", + "serial_number": "I6332527703739342000020", + "firmware_version": "4.8.1.25341.2677643.1597353330", + "mac_address": "A8:1B:6A:53:6A:98", + "ip_address": "192.168.1.100", + "last_seen": "2024-01-20T15:30:00Z", + "is_legacy_id": false, + "capabilities": ["multiroom", "bluetooth", "aux"] + }, + "data_sources": { + "presets": "local", + "recents": "mirror_primary", + "sources": "local" + }, + "migration": { + "from_bose_account": "bose-acc-xyz", + "migrated_at": "2024-01-18T14:30:00Z", + "method": "gradual", + "rollback_available": true, + "data_preserved": ["presets", "recents", "sources"] + }, + "health": { + "status": "healthy", + "last_check": "2024-01-20T15:30:00Z", + "response_time": 45, + "connectivity": "online", + "error_count": 0 + } +} +``` + +### Event Log Format (events.log) +``` +# SoundTouch Service Event Log - Device A81B6A536A98 +# Format: TIMESTAMP|EVENT_ID|EVENT_TYPE|SOURCE|DATA_JSON +# Version: 1.0 + +2024-01-20T15:30:00.123Z|evt_12345|now_playing|websocket|{"source":"SPOTIFY","track":"Song Name","artist":"Artist Name","album":"Album Name"} +2024-01-20T15:30:30.456Z|evt_12346|volume_changed|websocket|{"volume":45,"muted":false,"previous_volume":40} +2024-01-20T15:31:00.789Z|evt_12347|preset_selected|websocket|{"preset":1,"source":"SPOTIFY","location":"spotify:track:123abc"} +2024-01-20T15:31:15.012Z|evt_12348|disparity_detected|mirror|{"endpoint":"/v1/presets","local_hash":"abc123","upstream_hash":"def456","severity":"medium"} +2024-01-20T15:32:00.345Z|evt_12349|health_check|system|{"response_time":42,"status":"healthy","connectivity":"online"} +``` + +### Disparity Log Format (disparities.log) +``` +# SoundTouch Service Disparity Log +# Format: TIMESTAMP|DISPARITY_ID|DEVICE_ID|ACCOUNT_ID|ENDPOINT|TYPE|SEVERITY|DETAILS_JSON +# Version: 1.0 + +2024-01-20T15:31:15.012Z|disp_12345|A81B6A536A98|acc_12345|/v1/presets|count_mismatch|medium|{"field_path":"preset_count","local_value":5,"upstream_value":4,"description":"Local has one additional preset"} +2024-01-20T15:32:45.678Z|disp_12346|A81B6A536A98|acc_12345|/v1/recents|timestamp_format|low|{"field_path":"recent[0].utc_time","local_value":"2024-01-20T15:30:00Z","upstream_value":"1705761000","description":"Timestamp format difference"} +2024-01-20T15:35:20.901Z|disp_12347|B92C7B647B09|acc_12345|/v1/account/full|structure_diff|high|{"field_path":"device[1].ip_address","local_value":"present","upstream_value":"missing","description":"IP address field missing in upstream response"} +``` + +## State Machine Definitions + +### Device State Transitions + +``` +Unregistered β†’ Discovered (via discovery) +Discovered β†’ Registering (via user action/auto-registration) +Registering β†’ Active (via successful registration) +Registering β†’ Error (via registration failure) +Active β†’ Migrating (via migration start) +Active β†’ Offline (via connectivity loss) +Migrating β†’ Active (via migration success) +Migrating β†’ Error (via migration failure) +Offline β†’ Active (via connectivity restored) +Error β†’ Active (via error resolution) +Any State β†’ Retired (via explicit retirement) +``` + +### State Transition Rules + +```go +var StateTransitionRules = map[DeviceState][]DeviceState{ + DeviceStateUnregistered: {DeviceStateDiscovered}, + DeviceStateDiscovered: {DeviceStateRegistering, DeviceStateOffline}, + DeviceStateRegistering: {DeviceStateActive, DeviceStateError}, + DeviceStateActive: {DeviceStateMigrating, DeviceStateOffline, DeviceStateRetired}, + DeviceStateMigrating: {DeviceStateActive, DeviceStateError}, + DeviceStateOffline: {DeviceStateActive, DeviceStateError, DeviceStateRetired}, + DeviceStateError: {DeviceStateActive, DeviceStateOffline, DeviceStateRetired}, + DeviceStateRetired: {}, // Terminal state +} +``` + +### Transition Triggers + +```go +type TransitionTrigger struct { + Event DeviceEventType + Condition func(*DeviceLifecycle, *DeviceEvent) bool + Target DeviceState + Reason string +} + +var TransitionTriggers = []TransitionTrigger{ + { + Event: EventTypeDeviceOnline, + Condition: isOfflineDevice, + Target: DeviceStateActive, + Reason: "connectivity_restored", + }, + { + Event: EventTypeDeviceOffline, + Condition: isActiveDevice, + Target: DeviceStateOffline, + Reason: "connectivity_lost", + }, + { + Event: EventTypeMigrationStart, + Condition: isActiveDevice, + Target: DeviceStateMigrating, + Reason: "migration_initiated", + }, + // ... additional triggers +} +``` + +## Event Processing + +### Event Queue Implementation + +```go +type EventQueue struct { + buffer chan DeviceEvent + processors []EventProcessor + storage EventStorage + config EventQueueConfig +} + +type EventQueueConfig struct { + BufferSize int `json:"buffer_size"` + ProcessorCount int `json:"processor_count"` + FlushInterval time.Duration `json:"flush_interval"` + RetryAttempts int `json:"retry_attempts"` + DeadLetterQueue bool `json:"dead_letter_queue"` +} + +type EventProcessor interface { + ProcessEvent(event DeviceEvent) error + CanHandle(eventType DeviceEventType) bool +} +``` + +### Event Processing Flow + +``` +Event Input β†’ Validation β†’ Queue β†’ Processing β†’ Storage β†’ Notification + ↓ ↓ ↓ ↓ ↓ ↓ + Websocket Schema Buffer Parallel Files Webhooks + Discovery Check Memory Workers Logs SSE + API Call Format Retry DB Metrics + System Enrich DLQ +``` + +### Event Retention Policy + +```go +type RetentionPolicy struct { + EventType DeviceEventType `json:"event_type"` + RetentionDays int `json:"retention_days"` + MaxCount int `json:"max_count"` + Compression bool `json:"compression"` +} + +var DefaultRetentionPolicies = []RetentionPolicy{ + {EventTypeNowPlaying, 7, 1000, true}, + {EventTypeVolumeChanged, 1, 100, false}, + {EventTypeDisparityFound, 30, 10000, true}, + {EventTypeMigrationStart, 365, -1, false}, // Keep forever + {EventTypeHealthCheck, 7, 1000, true}, +} +``` + +### Development Requirements + +#### KISS Principle (Keep It Simple, Stupid) +- Prioritize simplicity and readability over performance optimization +- Use standard Go idioms and patterns +- Avoid premature abstraction and optimization +- Build the simplest thing that works first + +#### Quality Gates +Every change must pass these checks: +- `golangci-lint run --fix` - no linting issues +- `go test ./...` - all tests pass with no failures +- Integration tests verify existing functionality intact +- Code coverage maintained or improved + +#### Testing Requirements +- Unit tests for all new functions +- Integration tests for modified workflows +- Regression tests for existing functionality +- Mock external dependencies appropriately + +### Simplicity-First Performance Approach + +| Aspect | Simple Approach | Optimization Only When Needed | +|--------|----------------|-------------------------------| +| Memory Usage | Direct file operations, minimal caching | Add caching if performance issues arise | +| CPU Usage | Synchronous processing initially | Add async processing if bottlenecks occur | +| Storage | Simple append operations | Add rotation/compression when files grow large | +| Networking | Reuse existing patterns | Optimize only if latency becomes problematic | + +## Quality Assurance + +### Testing Strategy + +#### Unit Testing +```go +// Example test structure +func TestAccountManager_CreateAccount(t *testing.T) { + tests := []struct { + name string + input CreateAccountRequest + want *Account + wantErr bool + }{ + { + name: "valid account creation", + input: CreateAccountRequest{Name: "Test Account"}, + want: &Account{Name: "Test Account", Status: "active"}, + wantErr: false, + }, + { + name: "empty name should fail", + input: CreateAccountRequest{Name: ""}, + want: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := manager.CreateAccount(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("CreateAccount() error = %v, wantErr %v", err, tt.wantErr) + return + } + // Additional assertions... + }) + } +} +``` + +#### Integration Testing +- Test with real file operations in temporary directories +- Verify HTTP endpoints work with actual HTTP requests +- Test interaction with existing WebSocket system +- Ensure existing XML endpoints remain functional + +#### Quality Gates +```bash +# Required before each commit +golangci-lint run --fix +go test ./... +go test -race ./... + +# Required before milestone completion +go test ./... -v -cover +go test -bench=. ./... +``` + +### Security Considerations + +#### Simple Security Model +- Reuse existing authentication mechanisms +- Basic input validation with standard Go validation +- Simple file permissions (0755 for directories, 0644 for files) +- No complex authorization initially - build incrementally + +#### Input Validation +```go +// Simple validation approach +func ValidateAccount(account *Account) error { + if account.Name == "" { + return errors.New("account name cannot be empty") + } + if len(account.Name) > 100 { + return errors.New("account name too long") + } + if account.Email != "" && !isValidEmail(account.Email) { + return errors.New("invalid email format") + } + return nil +} +``` + +## Error Handling + +### Error Categories + +```go +type ErrorCategory string +const ( + ErrorCategoryValidation ErrorCategory = "validation" + ErrorCategorySystem ErrorCategory = "system" + ErrorCategoryNetwork ErrorCategory = "network" + ErrorCategoryStorage ErrorCategory = "storage" + ErrorCategoryTimeout ErrorCategory = "timeout" + ErrorCategoryAuth ErrorCategory = "authentication" +) + +type ServiceError struct { + Code string `json:"code"` + Message string `json:"message"` + Category ErrorCategory `json:"category"` + Timestamp time.Time `json:"timestamp"` + Context map[string]interface{} `json:"context"` + Retryable bool `json:"retryable"` + Severity string `json:"severity"` +} +``` + +### Error Recovery Strategies + +1. **Transient Errors**: Exponential backoff retry (3 attempts) +2. **Storage Errors**: Graceful degradation with in-memory fallback +3. **Network Errors**: Circuit breaker pattern with fallback data +4. **Validation Errors**: Immediate response with detailed feedback +5. **System Errors**: Alerting and automatic recovery attempts + +### Error Response Format + +```json +{ + "error": { + "code": "DEVICE_NOT_FOUND", + "message": "Device with ID 'A81B6A536A98' not found in account 'acc_12345'", + "category": "validation", + "timestamp": "2024-01-20T15:30:00Z", + "context": { + "account_id": "acc_12345", + "device_id": "A81B6A536A98", + "request_id": "req_67890" + }, + "retryable": false, + "severity": "error" + }, + "request_id": "req_67890" +} +``` + +## Monitoring and Observability + +### Simple Monitoring Approach + +#### Basic Health Check +```go +// Simple health check implementation +type HealthStatus struct { + Status string `json:"status"` // "healthy", "warning", "error" + Timestamp time.Time `json:"timestamp"` + Version string `json:"version"` + Uptime string `json:"uptime"` +} + +func (s *Server) HandleHealthCheck(w http.ResponseWriter, r *http.Request) { + health := HealthStatus{ + Status: "healthy", + Timestamp: time.Now(), + Version: version, + Uptime: time.Since(startTime).String(), + } + + // Simple checks + if !s.canWriteToDataDir() { + health.Status = "error" + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(health) +} +``` + +#### Leverage Existing Systems +- Extend existing parity mismatch logging for disparity detection +- Reuse existing interaction recording for request/response tracking +- Build upon current discovery and migration event logging +- Use existing WebSocket event system for device state changes + +#### Simple Metrics +```go +// Basic counters - no complex metrics initially +type SimpleMetrics struct { + AccountsCreated int `json:"accounts_created"` + DevicesActive int `json:"devices_active"` + EventsProcessed int `json:"events_processed_today"` + LastUpdate time.Time `json:"last_update"` +} + +// Update metrics in simple text file +func (m *SimpleMetrics) Save(dataDir string) error { + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dataDir, "metrics.json"), data, 0644) +} +``` + +This technical specification provides comprehensive details for implementing the enhanced state management system while maintaining compatibility with existing SoundTouch service functionality and meeting the performance requirements for small hardware deployments. \ No newline at end of file diff --git a/docs/concepts/upstream-service-simulation.md b/docs/concepts/upstream-service-simulation.md new file mode 100644 index 0000000..d4ad458 --- /dev/null +++ b/docs/concepts/upstream-service-simulation.md @@ -0,0 +1,393 @@ +# Upstream Bose Service Simulation - State Management Concept + +## Overview + +This document outlines the concept for simulating and replacing upstream Bose services with enhanced state management capabilities. The goal is to create a comprehensive local replacement that can handle device lifecycles, account management, and state synchronization while maintaining compatibility with existing SoundTouch devices. + +## Use Cases + +### Case 0: Account Management +- **Explicit Account Creation**: Accounts must be created through deliberate action (web UI, API call) +- **Mirror-Enhanced Creation**: Account creation can be enriched using mirrored data from upstream Bose endpoints when devices make requests +- **Data Recording**: Passively record account information during normal device operations for future use + +### Case 1a: Fresh Device Registration +- Initial setup/registration of a factory-reset or new device +- Device has no prior Bose account association +- Full local initialization with default configurations + +### Case 1b: Device Migration from Bose Account +- Migrate existing registered device from Bose services to local management +- Preserve existing device data (presets, recents, sources) +- Support gradual migration while maintaining Bose compatibility +- Mirror Bose account data for seamless transition + +### Case 2: Device Lifecycle and State Management +- Track and manage device lifecycle states and activities +- Maintain internal state based on incoming events from devices +- Detect disparities between local and upstream behavior +- Provide visibility into state changes and system health + +## Architecture Principles + +### 1. **Text-Based Storage for Debugging** +- Maintain all state in human-readable text formats (XML, JSON, plain text) +- Use small, focused files for each data aspect +- Enable easy debugging and manual inspection +- Optimize for small hardware deployments (Raspberry Pi Zero 2W) + +### 2. **Mirror-First Strategy** +- Keep mirror functionality active as long as possible +- Primary source switches from upstream to local only during: + - Explicit migration + - Sufficient local data accumulation + - Upstream service unavailability +- Record and mirror as much data as possible, even if not immediately used + +### 3. **Disparity Detection** +- Track differences between local and upstream responses +- Log discrepancies for analysis and improvement +- Provide visibility into implementation gaps +- Support parity testing and validation + +### 4. **Event-Driven State Management** +- Process device events asynchronously +- Track comprehensive event history in text files +- Support event replay and analysis +- Minimize noise while capturing important state changes + +## Enhanced Data Structure + +### Account Management + +``` +data/ +β”œβ”€β”€ accounts/ +β”‚ β”œβ”€β”€ {account-id}/ +β”‚ β”‚ β”œβ”€β”€ account.json # Account metadata +β”‚ β”œβ”€β”€ account-events.log # High-level account behavior tracking +β”‚ β”‚ β”œβ”€β”€ devices/ +β”‚ β”‚ β”‚ └── {device-id}/ +β”‚ β”‚ β”‚ β”œβ”€β”€ lifecycle.json # Device state and history +β”‚ β”‚ β”‚ β”œβ”€β”€ info.xml # Device information +β”‚ β”‚ β”‚ β”œβ”€β”€ presets.xml # Device presets +β”‚ β”‚ β”‚ β”œβ”€β”€ recents.xml # Recent plays +β”‚ β”‚ β”‚ β”œβ”€β”€ sources.xml # Configured sources +β”‚ β”‚ β”‚ └── events.log # Device event history +β”‚ β”‚ └── sessions/ +β”‚ β”‚ └── {session-id}/ # Recorded interaction sessions +└── system/ + β”œβ”€β”€ discovery.log # Device discovery events + └── migration.log # Migration activities +``` + +### Account Metadata Format + +```json +{ + "id": "account-12345", + "name": "User Account", + "email": "user@example.com", + "created_at": "2024-01-15T10:30:00Z", + "updated_at": "2024-01-20T15:45:00Z", + "status": "active", + "device_count": 3, + "migration_status": { + "started_at": "2024-01-18T09:00:00Z", + "devices_migrated": 1, + "devices_pending": 2, + "mirror_active": true + }, + "bose_account_id": "bose-original-id", + "data_sources": { + "local": true, + "bose_mirror": true, + "primary": "bose" + } +} +``` + +### Device Lifecycle Format + +```json +{ + "device_id": "A81B6A536A98", + "account_id": "account-12345", + "state": "active", + "created_at": "2024-01-15T10:30:00Z", + "updated_at": "2024-01-20T16:22:00Z", + "state_history": [ + { + "from": "unregistered", + "to": "registering", + "timestamp": "2024-01-15T10:30:00Z", + "reason": "fresh_device_setup", + "source": "discovery" + }, + { + "from": "registering", + "to": "active", + "timestamp": "2024-01-15T10:35:00Z", + "reason": "registration_complete", + "source": "system" + } + ], + "metadata": { + "name": "Living Room Speaker", + "type": "SoundTouch 30", + "serial_number": "I6332527703739342000020", + "firmware_version": "4.8.1.25341.2677643.1597353330", + "mac_address": "A8:1B:6A:53:6A:98", + "ip_address": "192.168.1.100", + "last_seen": "2024-01-20T16:20:00Z", + "is_legacy_id": false + }, + "data_sources": { + "presets": "local", + "recents": "mirror_primary", + "sources": "local" + }, + "migration": { + "from_bose_account": "bose-account-xyz", + "migrated_at": "2024-01-18T14:30:00Z", + "method": "gradual", + "rollback_available": true + } +} +``` + +### Event Log Format + +``` +# Device Events Log - A81B6A536A98 +# Format: TIMESTAMP|EVENT_TYPE|SOURCE|DATA + +2024-01-20T16:15:00Z|now_playing|websocket|{"source":"SPOTIFY","track":"Song Name","artist":"Artist Name"} +2024-01-20T16:15:30Z|volume_changed|websocket|{"volume":45,"muted":false} +2024-01-20T16:16:00Z|preset_selected|websocket|{"preset":1,"source":"SPOTIFY","location":"spotify:track:123"} +2024-01-20T16:18:00Z|disparity_detected|mirror|{"endpoint":"/v1/account/full","local_hash":"abc123","upstream_hash":"def456"} +2024-01-20T16:20:00Z|device_online|discovery|{"ip":"192.168.1.100","method":"mdns"} +``` + +### Disparity Log Format + +``` +# Parity Analysis Log +# Format: TIMESTAMP|ENDPOINT|DEVICE|ACCOUNT|DISPARITY_TYPE|DETAILS + +2024-01-20T16:18:00Z|/v1/account/full|A81B6A536A98|account-12345|content_mismatch|preset_count:local=5,upstream=4 +2024-01-20T16:19:15Z|/v1/presets|A81B6A536A98|account-12345|xml_structure|missing_container_art_in_local +2024-01-20T16:20:30Z|/v1/recents|A81B6A536A98|account-12345|timestamp_format|local=RFC3339,upstream=custom +``` + +## Implementation Strategy + +### Phase 1: Enhanced State Tracking + +1. **Account Management Service** + - Explicit account creation API + - Mirror-enhanced account initialization + - Account status and migration tracking + +2. **Device Lifecycle Manager** + - Comprehensive state machine for device lifecycle + - Event-driven state transitions + - Text-based state persistence + +3. **Enhanced Mirror System** + - Extended mirroring with disparity detection + - Selective data source switching + - Parity analysis and logging + +### Phase 2: Gradual Migration Support + +1. **Migration Controller** + - Device-by-device migration orchestration + - Rollback capability with state preservation + - Migration progress tracking + +2. **Dual-Source Data Management** + - Smart routing between local and upstream data + - Graceful fallback mechanisms + - Data source preference management + +3. **State Synchronization** + - Bidirectional sync capabilities + - Conflict resolution strategies + - Sync status monitoring + +### Phase 3: Advanced Analytics + +1. **Disparity Analysis Engine** + - Automated disparity detection and classification + - Trend analysis and reporting + - Implementation gap identification + +2. **System Health Monitoring** + - Device connectivity monitoring + - Service availability tracking + - Performance metrics collection + +3. **Data Export and Backup** + - Account data export for migration + - Incremental backup strategies + - Data integrity verification + +## API Enhancements + +### Account Management APIs + +```http +# Create account explicitly +POST /api/v1/accounts +Content-Type: application/json + +{ + "name": "User Account", + "email": "user@example.com" +} + +# Get account with migration status +GET /api/v1/accounts/{account-id} + +# Initiate account migration from Bose +POST /api/v1/accounts/{account-id}/migrate +Content-Type: application/json + +{ + "bose_account_id": "bose-original-id", + "strategy": "gradual" +} +``` + +### Device Lifecycle APIs + +```http +# Register fresh device +POST /api/v1/accounts/{account-id}/devices +Content-Type: application/json + +{ + "device_id": "A81B6A536A98", + "name": "Living Room Speaker", + "registration_type": "fresh" +} + +# Get device state and lifecycle +GET /api/v1/accounts/{account-id}/devices/{device-id}/state + +# Migrate device from Bose account +POST /api/v1/accounts/{account-id}/devices/{device-id}/migrate +Content-Type: application/json + +{ + "from_bose_account": "bose-account-xyz", + "preserve_data": true +} +``` + +### Monitoring and Analysis APIs + +```http +# Get disparity analysis +GET /api/v1/system/disparities?since=2024-01-20T00:00:00Z + +# Get migration status +GET /api/v1/system/migration/status + +# Export account data +GET /api/v1/accounts/{account-id}/export +``` + +## Integration with Existing Services + +### Enhanced Marge Service + +- Integrate lifecycle information into account responses +- Add migration status to device listings +- Support dual-source data routing +- Include disparity metadata in responses + +### Enhanced BMX Service + +- Track content source preferences by account +- Mirror and compare content recommendations +- Log streaming behavior for analysis +- Support gradual source migration + +### Discovery Service Integration + +- Link discovered devices to lifecycle manager +- Trigger lifecycle state transitions on discovery events +- Support both fresh registration and migration flows +- Handle legacy device ID migration automatically + +## Performance Considerations + +### Simplicity First (KISS Principle) + +- Favor simple, readable code over premature optimization +- Use straightforward algorithms and data structures +- Minimize complexity in favor of maintainability +- Build incrementally with small, testable changes + +### Quality Assurance + +- Complete test coverage for all new functionality +- Comprehensive linting with `golangci-lint run --fix` +- Full test suite execution `go test ./...` for each milestone +- Integration tests with existing functionality + +### File Management + +- Simple line-based append operations for logs +- Basic log rotation when needed +- Direct file operations without complex caching +- Straightforward data persistence + +## Development Principles + +### KISS (Keep It Simple, Stupid) +- Prioritize simplicity and readability over performance optimization +- Use standard Go idioms and patterns +- Avoid premature abstraction and optimization +- Build the simplest thing that works first + +### Quality First +- Every milestone must pass `golangci-lint run --fix` without issues +- Complete test suite must pass `go test ./...` before proceeding +- Integration tests ensure existing functionality remains intact +- Code coverage should be maintained or improved + +### Incremental Development +- Make small, focused changes that can be easily reviewed +- Each step should be independently testable and valuable +- Maintain backward compatibility throughout development +- Enable rollback at any point in the process + +### Leverage Existing Systems +- Reuse existing interaction recording for request/response tracking +- Build upon current parity mismatch detection system +- Extend existing datastore and handler patterns +- Integrate with established discovery and migration workflows + +## Future Enhancements + +Future improvements should maintain the simplicity-first approach: + +1. **Enhanced Web Interface** + - Simple dashboard for account and device management + - Basic migration progress tracking + - Straightforward device health monitoring + +2. **Extended Logging** + - Additional high-level behavior tracking + - Simple analytics based on existing parity data + - Enhanced debugging information + +3. **Community Integration** + - Standardized data export formats + - Simple reporting mechanisms + - Clear documentation for community contributions + +This concept provides a solid, maintainable foundation for replacing Bose's upstream services. The emphasis on simplicity, existing system reuse, and comprehensive testing ensures reliable functionality while maintaining the debugging capabilities needed for small hardware deployments. \ No newline at end of file diff --git a/docs/device-lifecycle-and-power-on-enhancement.md b/docs/device-lifecycle-and-power-on-enhancement.md new file mode 100644 index 0000000..22acfbd --- /dev/null +++ b/docs/device-lifecycle-and-power-on-enhancement.md @@ -0,0 +1,391 @@ +# Device Lifecycle and /power_on Enhancement + +## Overview + +This document provides a comprehensive analysis of the current SoundTouch device registration and lifecycle management implementation, and proposes enhancements using the `/power_on` endpoint to reduce dependency on local network connectivity. + +## Current Implementation Assessment + +### Device Information Sources + +The current system uses multiple data collection methods to build a complete device profile: + +#### 1. UPnP/SSDP Discovery +- **Protocol**: Multicast UDP discovery for `urn:schemas-upnp-org:service:SoundTouch:1` +- **Network Scope**: Limited to same network segment +- **Data Collected**: + ```go + type DiscoveredDevice struct { + Name string // From UPnP friendlyName + Host string // IP address + Port int // Usually 8090 + ModelID string // From UPnP modelName + SerialNo string // MAC address from UPnP + UPnPLocation string // Device description URL + UPnPUSN string // Unique service name + } + ``` + +#### 2. mDNS/Bonjour Discovery +- **Protocol**: Multicast DNS for `_soundtouch._tcp` services +- **Network Scope**: Limited to same network segment +- **Purpose**: Complements UPnP discovery with hostname resolution + +#### 3. `/info` Endpoint Enrichment +- **Protocol**: HTTP GET to `http://device:8090/info` +- **Network Scope**: Requires direct connectivity to device +- **Data Collected**: + ```xml + + My SoundTouch Device + SoundTouch 10 + 3230304 + + + SCM + 27.0.6.46330.5043500... + I6332527703739342000020 + + + https://streaming.bose.com + + AA:BB:CC:DD:EE:FF + 192.168.1.10 + + sm2 + rhino + GB + + ``` + +### Current Data Flow + +```mermaid +sequenceDiagram + participant Service as SoundTouch Service + participant UPnP as UPnP Discovery + participant mDNS as mDNS Discovery + participant Device as SoundTouch Device + participant DataStore as Data Store + participant User as User/App + + Note over Service,User: Current Device Registration Flow + + Service->>UPnP: Start SSDP Discovery + Service->>mDNS: Start mDNS Discovery + + UPnP->>UPnP: Send M-SEARCH multicast + Device->>UPnP: Respond with location URL + UPnP->>Device: Fetch device description XML + Device->>UPnP: Return basic device info + + mDNS->>mDNS: Query _soundtouch._tcp + Device->>mDNS: Respond with service info + + Service->>Service: Merge discovery results + Service->>Device: GET /info (enrich data) + Device->>Service: Return detailed device info + Service->>DataStore: Store discovered device + + Note over User,DataStore: User Registration + User->>Service: POST /account/{id}/devices + Note right of User: deviceId + user-friendly name + Service->>DataStore: Link device to account + + Note over Service,DataStore: Migration Process + Service->>Device: GET /info (device identification) + Device->>Service: Return device details + Service->>Service: Build migration summary + Service->>Device: Apply configuration changes +``` + +### Device Registration Points + +The system has distinct phases where device information is collected and enhanced: + +#### Phase 1: Discovery (Network-Dependent) +**Trigger**: Automatic network scanning +**Data Sources**: UPnP + mDNS + `/info` endpoint +**Limitations**: ❌ Requires same network segment + +#### Phase 2: User Registration (User-Controlled) +**Trigger**: User adds device to account +**Endpoint**: `POST /streaming/account/{accountId}/devices` +**Request Format**: +```xml + + Living Room Speaker + +``` +**Data Added**: βœ… User-friendly name, Account association + +#### Phase 3: Ongoing Updates (Mixed) +**Triggers**: Device state changes, firmware updates, network changes +**Methods**: Periodic `/info` polling, Discovery refresh, User configuration + +### Current Data Model + +The system maintains comprehensive device information: + +```go +type ServiceDeviceInfo struct { + DeviceID string `json:"device_id"` // MAC or UUID + Name string `json:"name"` // User-friendly name + ProductCode string `json:"product_code"` // Device model + DeviceSerialNumber string `json:"device_serial_number"` // Hardware serial + ProductSerialNumber string `json:"product_serial_number"` // Product serial + FirmwareVersion string `json:"firmware_version"` // Software version + IPAddress string `json:"ip_address"` // Current IP + MacAddress string `json:"mac_address"` // MAC address + AccountID string `json:"account_id"` // Account association + DiscoveryMethod string `json:"discovery_method"` // How discovered +} +``` + +## Limitations of Current Approach + +### Network Dependency Issues + +| Issue | Impact | Affected Operations | +|-------|--------|-------------------| +| **Same Network Requirement** | High | Device discovery, Initial setup | +| **Direct Connectivity Need** | High | Device enrichment, Migration | +| **Firewall/NAT Restrictions** | Medium | Corporate networks, Complex setups | +| **Multi-VLAN Environments** | High | Enterprise deployments | +| **Remote Management** | Critical | Off-site device support | + +### Service Architecture Limitations + +1. **Geographic Constraints**: Service must be deployed on same network as speakers +2. **Scalability Issues**: Cannot centralize device management across multiple locations +3. **Discovery Reliability**: Multicast protocols can be unreliable in complex networks +4. **Real-time Updates**: No device-initiated communication for state changes + +## /power_on Enhancement Proposal + +### Current /power_on Request Analysis + +The `/power_on` endpoint receives comprehensive device data that could replace many network-dependent operations: + +```xml + + + I6332527703739342000020 + 27.0.6.46330.5043500 epdbuild.trunk.hepdswbld04.2022-08-04T11:20:29 + + 069231P63364828AE + + + + + Excellent + 192.168.178.1 + + A81B6A536A98 + A81B6A849D99 + + 192.168.178.35 + Wireless + + + + + + +``` + +### Data Completeness Comparison + +| Data Field | Current `/info` | `/power_on` | Gap Assessment | +|------------|----------------|-------------|----------------| +| **Device ID** | βœ… UUID format | βœ… MAC format | Different format | +| **Device Name** | βœ… Internal name | ❌ Missing | **Critical Gap** | +| **Device Type** | βœ… Model string | βœ… Product code | βœ… Available | +| **Account ID** | βœ… marge UUID | ❌ Missing | **Critical Gap** | +| **Service URL** | βœ… marge URL | ❌ Missing | **Important Gap** | +| **Firmware Version** | βœ… Full version | βœ… Full version | βœ… Available | +| **Serial Numbers** | βœ… Component serials | βœ… Device + Product | βœ… Available | +| **MAC Addresses** | βœ… Interface-specific | βœ… Multiple MACs | βœ… Enhanced | +| **IP Address** | βœ… Interface IPs | βœ… Current IP | βœ… Available | +| **Network Status** | ❌ Basic | βœ… Rich diagnostics | βœ… **Enhanced** | +| **Regional Settings** | βœ… Country/Region | ❌ Missing | **Important Gap** | + +### Enhancement Benefits + +#### 1. Network Independence +- βœ… Works across internet/WAN connections +- βœ… No multicast/broadcast requirements +- βœ… Firewall/NAT friendly +- βœ… Supports remote device management + +#### 2. Real-time Device State +- βœ… Device-initiated communication +- βœ… Power-on event notifications +- βœ… Network status updates +- βœ… Firmware change detection + +#### 3. Enhanced Diagnostics +- βœ… Signal strength (RSSI) +- βœ… Gateway information +- βœ… Connection type details +- βœ… Real-time network status + +### Implementation Strategy + +#### Phase 1: Hybrid Approach +Implement `/power_on` processing while maintaining existing discovery methods: + +```go +func (s *Server) HandleMargePowerOn(w http.ResponseWriter, r *http.Request) { + // Parse power_on request + var powerOnData models.CustomerSupportRequest + if err := xml.Unmarshal(body, &powerOnData); err != nil { + // Fallback to existing discovery + return s.fallbackToDiscovery(r.RemoteAddr) + } + + // Extract device information + deviceMAC := powerOnData.Device.ID + deviceIP := powerOnData.DiagnosticData.DeviceLandscape.IPAddress + + // Lookup existing device data + deviceInfo := s.lookupDeviceByMAC(deviceMAC) + if deviceInfo == nil { + // New device - trigger registration flow + deviceInfo = s.createDeviceFromPowerOn(powerOnData) + } + + // Update with power_on data + s.updateDeviceFromPowerOn(deviceInfo, powerOnData) + + // Determine response actions + response := s.buildPowerOnResponse(deviceInfo) + s.sendResponse(w, response) +} +``` + +#### Phase 2: Gap Resolution +Address missing data through complementary mechanisms: + +1. **User-Friendly Names**: Maintain registration process for name assignment +2. **Account Association**: Enhance registration to link MAC addresses to accounts +3. **Service URLs**: Implement account-based service URL resolution +4. **Regional Settings**: Use IP geolocation or account preferences + +#### Phase 3: Enhanced Device Lifecycle + +```mermaid +sequenceDiagram + participant Device as SoundTouch Device + participant Service as SoundTouch Service + participant DataStore as Data Store + participant User as User/App + + Note over Device,User: Enhanced Device Lifecycle + + rect rgb(248, 255, 248) + Note over Device,DataStore: 1. Power-On Registration + Device->>Service: POST /power_on (rich device data) + Service->>DataStore: Lookup device by MAC + alt Device Unknown + Service->>DataStore: Create device record + Service->>User: Notify new device found + else Device Known + Service->>DataStore: Update device status + end + Service->>Device: Configuration response + end + + rect rgb(255, 248, 240) + Note over User,DataStore: 2. User Registration (Optional) + User->>Service: POST /setup/devices (name + preferences) + Service->>DataStore: Add user metadata to device + Service->>Device: Updated configuration (on next power_on) + end + + rect rgb(240, 248, 255) + Note over Device,DataStore: 3. Ongoing Updates + Device->>Service: POST /power_on (status changes) + Service->>Service: Detect firmware/network changes + Service->>DataStore: Update device record + alt Migration Needed + Service->>Device: Migration instructions + Device->>Device: Apply configuration + Device->>Service: POST /power_on (confirm changes) + end + end + + rect rgb(255, 248, 255) + Note over Service,User: 4. Remote Management + User->>Service: Management request (any location) + Service->>DataStore: Lookup device status + Service->>User: Current device state + Note over Service: No local network required + end +``` + +### Migration Strategy + +#### Current Migration Flow Issues +- Requires `/info` endpoint access for device identification +- Must be on same network for configuration changes +- Limited to devices discoverable via UPnP/mDNS + +#### Enhanced Migration with /power_on + +1. **Device Identification**: Use MAC address from `/power_on` instead of IP-based `/info` +2. **Configuration Delivery**: Send migration instructions in `/power_on` response +3. **Status Confirmation**: Device confirms changes via subsequent `/power_on` requests +4. **Remote Capability**: Manage devices from any network location + +```go +type PowerOnResponse struct { + ConfigurationUpdates []ConfigUpdate `json:"configuration_updates,omitempty"` + MigrationInstructions *Migration `json:"migration,omitempty"` + RegistrationRequired bool `json:"registration_required,omitempty"` +} + +type Migration struct { + Method string `json:"method"` // xml, hosts, resolv_conf + TargetURL string `json:"target_url"` + ProxyURL string `json:"proxy_url,omitempty"` + Options map[string]string `json:"options"` +} +``` + +## Recommendations + +### Immediate Actions (Phase 1) +1. **Enhance `/power_on` handler** to extract and store comprehensive device data +2. **Implement device lookup by MAC address** as primary identification method +3. **Create hybrid discovery system** using both `/power_on` and existing methods +4. **Add network-independent device management** capabilities + +### Medium-term Improvements (Phase 2) +1. **Implement account-device MAC mapping** for automatic association +2. **Add IP geolocation** for regional settings inference +3. **Create device registration UI** optimized for `/power_on` discovered devices +4. **Enhance migration system** to use `/power_on` response mechanism + +### Long-term Enhancements (Phase 3) +1. **Request firmware enhancement** to include missing data in `/power_on` +2. **Implement real-time device monitoring** via `/power_on` events +3. **Create centralized device management** independent of network topology +4. **Add predictive migration** based on device status patterns + +### Risk Mitigation +- **Maintain backward compatibility** with existing discovery methods +- **Implement graceful fallbacks** when `/power_on` data is incomplete +- **Preserve existing user workflows** while adding enhanced capabilities +- **Add comprehensive logging** for troubleshooting hybrid approach + +## Conclusion + +The `/power_on` endpoint provides a significant opportunity to reduce network dependencies while enhancing device management capabilities. By implementing a hybrid approach that leverages `/power_on` data for primary device identification and status updates while maintaining existing registration workflows for user-controlled metadata, the system can achieve: + +- **Network independence** for core device management +- **Enhanced real-time capabilities** through device-initiated communication +- **Improved scalability** across diverse network topologies +- **Better user experience** with automatic device discovery and status updates + +The proposed implementation strategy provides a clear path to achieve these benefits while maintaining system reliability and user workflow compatibility. \ No newline at end of file diff --git a/docs/device-lifecycle-summary.md b/docs/device-lifecycle-summary.md new file mode 100644 index 0000000..8ebda79 --- /dev/null +++ b/docs/device-lifecycle-summary.md @@ -0,0 +1,150 @@ +# Device Lifecycle Analysis - Executive Summary + +## Current State Assessment + +The SoundTouch service currently relies heavily on local network connectivity for device discovery and management: + +### βœ… Strengths +- **Comprehensive device data** through `/info` endpoint +- **Robust discovery** via UPnP/SSDP + mDNS +- **User-controlled registration** with friendly names +- **Complete device lifecycle management** + +### ❌ Limitations +- **Network dependency**: Requires same network segment for discovery +- **Geographic constraints**: Service must be co-located with devices +- **Firewall/NAT issues**: Multicast protocols unreliable in complex networks +- **No remote management**: Cannot manage devices from external networks + +## /power_on Enhancement Opportunity + +The `/power_on` endpoint provides rich device data that could eliminate network dependencies: + +### Current /power_on Data +```xml + + + I6332527703739342000020 + 27.0.6.46330.5043500... + + 069231P63364828AE + + + + + Excellent + 192.168.178.1 + + A81B6A536A98 + A81B6A849D99 + + 192.168.178.35 + Wireless + + + +``` + +### Missing Data Gaps +| Data | Current Source | Available in /power_on | Impact | +|------|----------------|----------------------|---------| +| **User-friendly name** | Registration | ❌ Missing | **High** - UI/UX | +| **Account association** | Registration | ❌ Missing | **Critical** - Authorization | +| **Service URLs** | `/info` | ❌ Missing | **High** - Migration | +| **Regional settings** | `/info` | ❌ Missing | **Medium** - Localization | + +## Recommended Implementation Strategy + +### Phase 1: Hybrid Enhancement (Immediate) +- **Enhance `/power_on` handler** to process full device data +- **Implement MAC-based device lookup** for identification +- **Maintain existing registration flow** for user metadata +- **Add network-independent capabilities** as primary features + +```go +// Enhanced flow +Device -> POST /power_on -> Service identifies by MAC -> Update/Create device record +``` + +### Phase 2: Gap Resolution (Short-term) +- **Account-device MAC mapping** for automatic association +- **IP geolocation** for regional settings inference +- **Registration UI optimization** for /power_on discovered devices +- **Migration via response payload** instead of direct device access + +### Phase 3: Full Network Independence (Medium-term) +- **Centralized device management** across multiple networks +- **Real-time device monitoring** via /power_on events +- **Predictive migration** based on device status patterns +- **Enhanced firmware integration** with additional /power_on data + +## Key Benefits + +### βœ… Immediate Gains +- **Network independence**: Manage devices from any location +- **Real-time updates**: Device-initiated status reporting +- **Enhanced diagnostics**: Signal strength, connection type, network status +- **Simplified deployment**: No multicast/broadcast requirements + +### βœ… Long-term Advantages +- **Scalable architecture**: Centralized management across sites +- **Improved reliability**: Eliminates discovery protocol dependencies +- **Better user experience**: Automatic device detection and status +- **Future-proof design**: Device-driven communication model + +## Implementation Approach + +### Hybrid Strategy +```mermaid +graph TD + PowerOn[Device /power_on] --> Identify[MAC-based Identification] + Identify --> New{New Device?} + + New -->|Yes| Create[Create Device Record] + New -->|No| Update[Update Existing Record] + + Create --> CheckAccount{Account Known?} + CheckAccount -->|No| RegisterFlow[Trigger Registration] + CheckAccount -->|Yes| LinkAccount[Link to Account] + + Update --> DetectChanges[Detect Changes] + DetectChanges --> Migration{Migration Needed?} + Migration -->|Yes| SendInstructions[Send Migration Instructions] + + LinkAccount --> Response[Send Configuration Response] + SendInstructions --> Response + RegisterFlow --> Response +``` + +### Risk Mitigation +- **Maintain backward compatibility** with existing discovery +- **Graceful fallbacks** when /power_on data incomplete +- **Preserve user workflows** while adding enhanced capabilities +- **Comprehensive logging** for troubleshooting + +## Success Metrics + +### Technical Metrics +- **Network independence**: % of operations not requiring local network +- **Real-time capability**: Power-on event processing latency < 2s +- **Data completeness**: % of devices with full metadata via /power_on +- **Migration success**: % of successful remote migrations + +### User Experience Metrics +- **Discovery reliability**: % of devices automatically detected +- **Setup time**: Time from device power-on to full management +- **Management accessibility**: % of operations available remotely +- **Error reduction**: Decrease in network-related issues + +## Conclusion + +The `/power_on` enhancement represents a strategic opportunity to: + +1. **Eliminate network dependencies** while maintaining full functionality +2. **Enable remote device management** across diverse network topologies +3. **Improve user experience** through automatic device detection +4. **Future-proof the architecture** for scalable device management + +**Recommendation**: Proceed with hybrid implementation approach, prioritizing network independence while preserving existing user workflows and system reliability. + +**Timeline**: Phase 1 implementation feasible within 2-3 sprints, with Phases 2-3 extending capabilities based on user feedback and firmware enhancement opportunities. \ No newline at end of file diff --git a/docs/diagrams/migration-flow.md b/docs/diagrams/migration-flow.md new file mode 100644 index 0000000..e19471b --- /dev/null +++ b/docs/diagrams/migration-flow.md @@ -0,0 +1,222 @@ +# Migration Flow Diagrams + +This document specifies the diagrams needed for the migration guide, with descriptions that can be used to create actual visual diagrams. + +## 1. Overall Migration Process Flow + +### Description +A flowchart showing the complete migration journey from start to finish. + +### Elements +``` +[Start] β†’ [Install SoundTouch Service] β†’ [Create Account] β†’ [Prepare Devices] + ↓ +[Enable Remote Services] β†’ [Discover Devices] β†’ [Register Devices] + ↓ +[Start Migration] β†’ [Data Collection Phase] β†’ [Testing Phase] β†’ [Full Local Phase] + ↓ +[Verify Migration] β†’ [Complete] β†’ [Post-Migration Setup] +``` + +### Decision Points +- Multiple devices? β†’ Repeat device steps +- Migration issues? β†’ Rollback option +- All devices complete? β†’ Account fully migrated + +### Color Coding +- **Blue**: Service setup steps +- **Green**: Successful completion states +- **Orange**: In-progress/testing states +- **Red**: Error handling/rollback paths +- **Gray**: Optional steps + +## 2. Network Topology Diagram + +### Description +Shows the network layout with Raspberry Pi, router, and SoundTouch devices. + +### Components +``` +Internet Cloud + ↑↓ (Optional - during migration) +Home Router (192.168.1.1) + β”œβ”€β”€ Raspberry Pi (192.168.1.10) [SoundTouch Service] + β”œβ”€β”€ Living Room Speaker (192.168.1.100) + β”œβ”€β”€ Kitchen Speaker (192.168.1.101) + β”œβ”€β”€ Bedroom Speaker (192.168.1.102) + └── Office Speaker (192.168.1.103) +``` + +### Connections +- **Solid lines**: Active connections +- **Dashed lines**: Migration-phase connections to Bose cloud +- **Thick lines**: Primary data flow to local service + +## 3. Device State Lifecycle + +### Description +State machine showing device progression through migration phases. + +### States and Transitions +``` +[Unregistered] β†’ [Discovered] β†’ [Registered] β†’ [Migrating] + ↓ +[Active - Local Only] ← [Active - Testing] ← [Active - Data Collection] + ↑ ↓ +[Error/Rollback] ← ← ← ← ← ← ← ← ← ← ← ← ← ← ← ← ← [Migration Failed] +``` + +### State Descriptions +- **Unregistered**: Device not known to service +- **Discovered**: Found on network, remote services enabled +- **Registered**: Added to account, ready for migration +- **Migrating - Data Collection**: Building local database +- **Migrating - Testing**: Using local service with fallback +- **Active - Local Only**: Full independence achieved +- **Error/Rollback**: Issues detected, can revert to Bose + +## 4. Data Flow During Migration + +### Description +Shows how data flows between components during different migration phases. + +### Phase 1 - Data Collection +``` +SoundTouch Device β†’ Bose Cloud Services + ↓ (mirror) + Local Service (collecting data) +``` + +### Phase 2 - Testing +``` +SoundTouch Device ↔ Local Service (primary) + ↕ (fallback when needed) + Bose Cloud Services +``` + +### Phase 3 - Full Local +``` +SoundTouch Device ↔ Local Service (only) + +Bose Cloud Services (disconnected) +``` + +### Data Types +- **Presets**: Station favorites and custom sources +- **Recents**: Play history and recently accessed content +- **Sources**: Configured music services (Spotify, etc.) +- **Device Config**: Network settings, capabilities, metadata + +## 5. Migration Timeline Visualization + +### Description +Gantt-chart style timeline showing typical migration schedule. + +### Timeline (7-day example) +``` +Day 1-2: Data Collection Phase + β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ + +Day 3-4: Data Validation + β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ + +Day 5-6: Testing Phase + β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ + +Day 7+: Full Local Operation + β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ†’ +``` + +### Parallel Activities +- Multiple devices can be in different phases +- Service continues operating throughout +- User can interact normally during process + +## 6. Service Architecture Overview + +### Description +High-level architecture showing enhanced SoundTouch service components. + +### Components +``` +Web Dashboard ← β†’ HTTP API ← β†’ REST Endpoints + ↑ ↑ ↑ + └─── User ──────┼──── Devices β”€β”˜ + ↓ + Service Core + β”œβ”€β”€ Account Manager + β”œβ”€β”€ Device Lifecycle + β”œβ”€β”€ Event Processor + β”œβ”€β”€ Migration Controller + └── Data Store + ↓ + File System Storage + β”œβ”€β”€ accounts/ + β”œβ”€β”€ devices/ + β”œβ”€β”€ sessions/ (existing) + └── system/ +``` + +### External Integrations +- **Bose Cloud** (during migration) +- **Music Services** (Spotify, TuneIn, etc.) +- **Discovery Services** (mDNS, UPnP) + +## 7. Error Handling and Rollback Flow + +### Description +Decision tree for handling migration issues and rollback scenarios. + +### Error Detection +``` +Migration Issue Detected + β”œβ”€β”€ Device Unresponsive β†’ Retry β†’ Success/Rollback + β”œβ”€β”€ Data Corruption β†’ Restore from Backup β†’ Continue/Rollback + β”œβ”€β”€ Service Unavailable β†’ Wait/Restart β†’ Continue/Rollback + └── User Dissatisfaction β†’ Manual Rollback β†’ Restore Bose Config +``` + +### Rollback Process +``` +[Rollback Initiated] + ↓ +[Disable Local Services] + ↓ +[Restore Original Device Config] + ↓ +[Re-enable Bose Services] + ↓ +[Verify Functionality] + ↓ +[Rollback Complete] +``` + +## Implementation Notes + +### For Diagram Creation +1. Use consistent colors as specified in main color scheme +2. Include clear labels for all components +3. Show directional flow with appropriate arrows +4. Use standard flowchart symbols where applicable +5. Ensure text is readable at various sizes + +### Tools Recommended +- **Lucidchart**: Professional flowcharts and network diagrams +- **Draw.io**: Free online diagram tool +- **Miro**: Collaborative whiteboarding +- **PlantUML**: Code-based diagram generation + +### File Naming Convention +- `migration-flow-overview.svg` - Overall process flow +- `network-topology.svg` - Network layout +- `device-lifecycle.svg` - State machine +- `data-flow-phases.svg` - Data flow during migration +- `migration-timeline.svg` - Timeline visualization +- `service-architecture.svg` - System architecture +- `error-rollback-flow.svg` - Error handling + +### Accessibility +- Include alt-text descriptions +- Use patterns/textures in addition to colors +- Ensure sufficient contrast +- Provide text-based versions for screen readers \ No newline at end of file diff --git a/docs/guides/IOT-IMPLEMENTATION-GUIDE.md b/docs/guides/IOT-IMPLEMENTATION-GUIDE.md new file mode 100644 index 0000000..aa2e287 --- /dev/null +++ b/docs/guides/IOT-IMPLEMENTATION-GUIDE.md @@ -0,0 +1,753 @@ +# IoT Implementation Guide + +## Overview + +This guide provides technical implementation details for integrating with the Bose SoundTouch IoT configuration system. It covers the AWS IoT Core integration, certificate management, and device shadow operations. + +## Prerequisites + +- AWS IoT Core account and permissions +- Understanding of MQTT protocol +- Knowledge of X.509 certificate management +- Familiarity with JSON and protobuf serialization + +## Architecture Components + +### Core System Design + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Mobile App β”‚ β”‚ Alexa Voice β”‚ β”‚ Web Interface β”‚ +β”‚ β”‚ β”‚ Assistant β”‚ β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ AWS IoT Core β”‚ + β”‚ (MQTT Broker + β”‚ + β”‚ Device Shadows) β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ MQTT/TLS + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ SoundTouch Device β”‚ + β”‚ β”‚ + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ + β”‚ β”‚ IoT Service β”‚ β”‚ + β”‚ β”‚ (/opt/Bose/IoT) β”‚ β”‚ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ + β”‚ β”‚ BoseApp Service β”‚ β”‚ + β”‚ β”‚ (/opt/Bose/BoseApp) β”‚ β”‚ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### Configuration Flow + +``` +1. Device Boot + β”‚ + β–Ό +2. Read IoT.xml (/mnt/nv/BoseApp-Persistence/1/IoT.xml) + β”‚ + β–Ό +3. Load Certificates (/mnt/nv/IoTCerts/) + β”‚ + β–Ό +4. Establish MQTT/TLS Connection + β”‚ + β–Ό +5. Subscribe to Device Shadow Topics + β”‚ + β–Ό +6. Publish Current Device State + β”‚ + β–Ό +7. Listen for Delta Messages +``` + +## Implementation Details + +### 1. Configuration File Management + +#### IoT.xml Structure +```xml + + +``` + +#### Loading Configuration (C++ Implementation) +```cpp +#include +#include + +struct IoTConfig { + std::string clientID; + std::string iotEndpoint; + std::string deployment; +}; + +IoTConfig loadIoTConfig(const std::string& configPath) { + std::ifstream file(configPath); + std::string content((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + + rapidxml::xml_document<> doc; + doc.parse<0>(&content[0]); + + auto configNode = doc.first_node("Configuration"); + + IoTConfig config; + config.clientID = configNode->first_attribute("clientID")->value(); + config.iotEndpoint = configNode->first_attribute("iotEndpoint")->value(); + config.deployment = configNode->first_attribute("deployment")->value(); + + return config; +} +``` + +### 2. Certificate Management + +#### Certificate Files Structure +``` +/mnt/nv/IoTCerts/ +β”œβ”€β”€ iot-cert.pem.crt # Device client certificate +β”œβ”€β”€ iot-private.pem.key # Device private key +└── default.pem # Additional cert data + +/var/lib/iot/ +└── rootCA.crt # AWS IoT Root CA +``` + +#### Certificate Registration Process +```cpp +#include +#include +#include + +class IoTCertificateManager { +private: + static const std::string CERT_ENDPOINT; + static const std::string CERT_PATH; + static const std::string KEY_PATH; + +public: + bool generateCSR() { + // Generate EC key pair + EC_KEY* eckey = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1); + EC_KEY_generate_key(eckey); + + // Create certificate request + X509_REQ* req = X509_REQ_new(); + X509_REQ_set_version(req, 0); + + // Set subject name + X509_NAME* name = X509_NAME_new(); + X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC, + (unsigned char*)clientID.c_str(), -1, -1, 0); + X509_REQ_set_subject_name(req, name); + + // Set public key + EVP_PKEY* pkey = EVP_PKEY_new(); + EVP_PKEY_set1_EC_KEY(pkey, eckey); + X509_REQ_set_pubkey(req, pkey); + + // Sign request + X509_REQ_sign(req, pkey, EVP_sha256()); + + return sendCSRToEndpoint(req, pkey); + } + + bool sendCSRToEndpoint(X509_REQ* req, EVP_PKEY* pkey) { + // Send CSR to voice.api.bose.io/alexa/certificate + // Receive certificate response + // Store certificate and private key + return true; + } +}; + +const std::string IoTCertificateManager::CERT_ENDPOINT = + "https://voice.api.bose.io/alexa/certificate"; +const std::string IoTCertificateManager::CERT_PATH = + "/mnt/nv/IoTCerts/iot-cert.pem.crt"; +const std::string IoTCertificateManager::KEY_PATH = + "/mnt/nv/IoTCerts/iot-private.pem.key"; +``` + +### 3. MQTT Connection Implementation + +#### AWS IoT SDK Integration +```cpp +#include +#include + +class IoTConnectionManager { +private: + std::unique_ptr mqttClient; + std::unique_ptr shadowClient; + IoTConfig config; + +public: + awsiotsdk::ResponseCode connect() { + // Setup connection parameters + std::string endpoint = config.iotEndpoint; + uint16_t port = 8883; // MQTT over SSL + + // Load certificates + std::string certPath = "/mnt/nv/IoTCerts/iot-cert.pem.crt"; + std::string keyPath = "/mnt/nv/IoTCerts/iot-private.pem.key"; + std::string rootCaPath = "/var/lib/iot/rootCA.crt"; + + // Create network connection + auto networkConnection = std::make_shared( + endpoint, port, rootCaPath, certPath, keyPath + ); + + // Create MQTT client + mqttClient = awsiotsdk::MqttClient::Create(networkConnection); + if (!mqttClient) { + return awsiotsdk::ResponseCode::FAILURE; + } + + // Connect with client ID + auto connectPacket = awsiotsdk::mqtt::ConnectPacket::Create( + config.clientID, + true, // cleanSession + awsiotsdk::mqtt::QoS::QOS0, + nullptr // will options + ); + + return mqttClient->Connect(std::chrono::milliseconds(5000), connectPacket); + } + + awsiotsdk::ResponseCode initializeShadow() { + shadowClient = awsiotsdk::Shadow::Create(mqttClient); + if (!shadowClient) { + return awsiotsdk::ResponseCode::FAILURE; + } + + // Subscribe to shadow delta + auto deltaHandler = [this](const std::string& thingName, + const std::string& payload) { + handleShadowDelta(thingName, payload); + }; + + return shadowClient->PerformUpdateAsync( + config.clientID, + "", // jsonString + deltaHandler, + std::chrono::seconds(10) + ); + } +}; +``` + +### 4. Device Shadow Operations + +#### Shadow Message Structures +```cpp +#include +#include +#include + +struct DeviceState { + std::string deviceState; // "CONNECTED" | "DISCONNECTED" + std::string powerState; // "ON" | "OFF" + std::string zoneState; // Zone configuration + std::string groupState; // Multi-room group info +}; + +class ShadowMessageBuilder { +public: + static std::string createReportedState(const DeviceState& state) { + rapidjson::Document doc; + doc.SetObject(); + auto& allocator = doc.GetAllocator(); + + // Create state object + rapidjson::Value stateObj(rapidjson::kObjectType); + rapidjson::Value reportedObj(rapidjson::kObjectType); + + // Add reported state fields + reportedObj.AddMember("deviceState", + rapidjson::Value(state.deviceState.c_str(), allocator), + allocator); + reportedObj.AddMember("powerState", + rapidjson::Value(state.powerState.c_str(), allocator), + allocator); + reportedObj.AddMember("zoneState", + rapidjson::Value(state.zoneState.c_str(), allocator), + allocator); + reportedObj.AddMember("groupState", + rapidjson::Value(state.groupState.c_str(), allocator), + allocator); + + stateObj.AddMember("reported", reportedObj, allocator); + doc.AddMember("state", stateObj, allocator); + + // Serialize to string + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc.Accept(writer); + + return buffer.GetString(); + } + + static DeviceState parseDesiredState(const std::string& json) { + rapidjson::Document doc; + doc.Parse(json.c_str()); + + DeviceState state; + if (doc.HasMember("state") && doc["state"].HasMember("desired")) { + auto& desired = doc["state"]["desired"]; + + if (desired.HasMember("powerState")) { + state.powerState = desired["powerState"].GetString(); + } + if (desired.HasMember("zoneState")) { + state.zoneState = desired["zoneState"].GetString(); + } + if (desired.HasMember("groupState")) { + state.groupState = desired["groupState"].GetString(); + } + } + + return state; + } +}; +``` + +#### Shadow Update Implementation +```cpp +class IoTShadowManager { +private: + std::shared_ptr shadowClient; + std::string thingName; + DeviceState currentState; + +public: + awsiotsdk::ResponseCode updateDeviceState(const DeviceState& newState) { + currentState = newState; + + std::string payload = ShadowMessageBuilder::createReportedState(newState); + + auto responseHandler = [](const std::string& thingName, + awsiotsdk::ShadowRequestType requestType, + awsiotsdk::ShadowResponseType responseType, + rapidjson::Document& payload) { + if (responseType == awsiotsdk::ShadowResponseType::Accepted) { + // Shadow update successful + std::cout << "Shadow updated successfully" << std::endl; + } else { + // Handle rejection + std::cout << "Shadow update rejected" << std::endl; + } + }; + + return shadowClient->PerformUpdateAsync( + thingName, + payload, + responseHandler, + std::chrono::seconds(10) + ); + } + + void handleShadowDelta(const std::string& thingName, + const std::string& payload) { + DeviceState desiredState = ShadowMessageBuilder::parseDesiredState(payload); + + // Apply desired state changes to device + if (!desiredState.powerState.empty()) { + applyPowerStateChange(desiredState.powerState); + } + + if (!desiredState.zoneState.empty()) { + applyZoneStateChange(desiredState.zoneState); + } + + if (!desiredState.groupState.empty()) { + applyGroupStateChange(desiredState.groupState); + } + + // Report updated state back to shadow + updateDeviceState(currentState); + } +}; +``` + +### 5. Service Integration + +#### Shepherd Service Configuration +```xml + + + + + /mnt/nv/BoseApp-Persistence/1/IoT.xml + /mnt/nv/IoTCerts + + + -c + /opt/Bose/etc/Voice.xml + + +``` + +#### System Startup Integration +```bash +#!/bin/bash +# /etc/init.d/SoundTouch fragment + +# Create IoT directories +mkdir -p /mnt/nv/BoseLog /mnt/nv/IoTCerts /mnt/nv/BoseApp-Persistence/1 +mkdir -m 700 -p /mnt/nv/BoseApp-Persistence/1/Keys + +# Set proper permissions for certificate storage +chmod 700 /mnt/nv/IoTCerts +chown iot:iot /mnt/nv/IoTCerts + +# Start shepherd daemon manager +shepherdd --config-dir /opt/Bose/etc --run-dir /var/run/shepherd +``` + +## Error Handling and Debugging + +### Connection Retry Logic +```cpp +class ConnectionRetryManager { +private: + int maxRetries = 10; + int retryDelaySeconds = 5; + +public: + awsiotsdk::ResponseCode connectWithRetry(IoTConnectionManager& manager) { + for (int attempt = 1; attempt <= maxRetries; ++attempt) { + std::cout << "Connection attempt " << attempt + << " to MQTT port at host " << config.iotEndpoint << std::endl; + + auto result = manager.connect(); + if (result == awsiotsdk::ResponseCode::SUCCESS) { + std::cout << "Successfully connected to MQTT server" << std::endl; + return result; + } + + std::cout << "MQTT port not available. Retrying in " + << retryDelaySeconds << " seconds" << std::endl; + + std::this_thread::sleep_for(std::chrono::seconds(retryDelaySeconds)); + retryDelaySeconds *= 2; // Exponential backoff + } + + std::cerr << "Failed to connect after " << maxRetries << " attempts" << std::endl; + return awsiotsdk::ResponseCode::FAILURE; + } +}; +``` + +### Logging and Monitoring +```cpp +class IoTLogger { +public: + static void logConnectionStatus(const std::string& status) { + std::cout << "[IoT] Connection status: " << status << std::endl; + } + + static void logShadowResponse(awsiotsdk::ShadowResponseType response, + const std::string& payload) { + if (response == awsiotsdk::ShadowResponseType::Accepted) { + std::cout << "[IoT] Shadow response: accepted. Payload: " << payload << std::endl; + } else { + std::cout << "[IoT] Shadow response: rejected" << std::endl; + } + } + + static void logCertificateStatus(bool success) { + if (success) { + std::cout << "[IoT] Certificate generated successfully" << std::endl; + } else { + std::cerr << "[IoT] Failed to generate iot certificate" << std::endl; + } + } +}; +``` + +## Testing and Validation + +### Unit Test Example +```cpp +#include + +class IoTConfigTest : public ::testing::Test { +protected: + void SetUp() override { + // Create test configuration file + std::ofstream file("/tmp/test_iot.xml"); + file << R"( +)"; + file.close(); + } +}; + +TEST_F(IoTConfigTest, LoadConfiguration) { + auto config = loadIoTConfig("/tmp/test_iot.xml"); + + EXPECT_EQ(config.clientID, "test-client-id"); + EXPECT_EQ(config.iotEndpoint, "test.iot.amazonaws.com"); + EXPECT_EQ(config.deployment, "TEST"); +} + +TEST_F(IoTConfigTest, ShadowMessageBuilder) { + DeviceState state; + state.deviceState = "CONNECTED"; + state.powerState = "ON"; + + std::string json = ShadowMessageBuilder::createReportedState(state); + + // Verify JSON contains expected fields + EXPECT_TRUE(json.find("\"deviceState\":\"CONNECTED\"") != std::string::npos); + EXPECT_TRUE(json.find("\"powerState\":\"ON\"") != std::string::npos); +} +``` + +## Security Best Practices + +1. **Certificate Management** + - Store private keys with 600 permissions + - Rotate certificates regularly + - Use hardware security modules when available + +2. **Network Security** + - Always use TLS 1.2 or higher + - Validate certificate chains + - Implement certificate pinning + +3. **Configuration Security** + - Encrypt sensitive configuration data + - Use secure storage for credentials + - Implement configuration validation + +## Troubleshooting Common Issues + +### Certificate Problems +```bash +# Check certificate validity +openssl x509 -in /mnt/nv/IoTCerts/iot-cert.pem.crt -text -noout + +# Verify private key matches certificate +openssl x509 -noout -modulus -in /mnt/nv/IoTCerts/iot-cert.pem.crt | openssl md5 +openssl rsa -noout -modulus -in /mnt/nv/IoTCerts/iot-private.pem.key | openssl md5 +``` + +### Connection Issues +```bash +# Test MQTT connectivity +mosquitto_pub -h a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com \ + -p 8883 --cafile /var/lib/iot/rootCA.crt \ + --cert /mnt/nv/IoTCerts/iot-cert.pem.crt \ + --key /mnt/nv/IoTCerts/iot-private.pem.key \ + -t '$aws/things/test/shadow/update' \ + -m '{"state":{"reported":{"test":"value"}}}' +``` + +### Service Debugging +```bash +# Check service status +ps aux | grep IoT + +# Monitor system logs +tail -f /mnt/nv/BoseLog/IoT.log + +# Check Shepherd status +shepherdd --status +``` + +## MQTT Monitoring and Research + +### Direct Device Credential Access + +With device certificates and private keys available from firmware backups, it's technically possible to monitor MQTT traffic: + +```bash +# Subscribe to your device's shadow events only +CLIENT_ID="577ecfcc-2db3-4989-92c9-76d7704f9fb3" # Your device's UUID +mosquitto_sub -h a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com \ + -p 8883 --cafile /var/lib/iot/rootCA.crt \ + --cert /mnt/nv/IoTCerts/iot-cert.pem.crt \ + --key /mnt/nv/IoTCerts/iot-private.pem.key \ + -t "\$aws/things/$CLIENT_ID/shadow/update/accepted" +``` + +### Security Constraints and Limitations + +#### AWS IoT Policy Restrictions +Device certificates are bound to restrictive policies: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "iot:Connect", + "Resource": "arn:aws:iot:us-east-1:*:client/${iot:ClientId}" + }, + { + "Effect": "Allow", + "Action": ["iot:Publish", "iot:Subscribe", "iot:Receive"], + "Resource": [ + "arn:aws:iot:us-east-1:*:topic/$aws/things/${iot:ClientId}/shadow/*" + ] + } + ] +} +``` + +**Limitations:** +- Access only to your specific device topics +- No wildcard subscriptions (`+` or `#`) +- No cross-device monitoring +- Potential IP geolocation restrictions +- Certificate revocation for unusual activity + +### Alternative Monitoring Approaches + +#### Network Traffic Capture (Recommended) +```bash +# Capture MQTT traffic patterns without authentication +tcpdump -i eth0 -s0 -w soundtouch_iot.pcap host a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com + +# Monitor connection patterns in real-time +tcpdump -i eth0 -n -A "host a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com and port 8883" + +# Extract timing and packet size information +tcpdump -i eth0 -ttt -s0 "host a2bhvr9c4wn4ya.iot.us-east-1.amazonaws.com" +``` + +#### Local MQTT Broker for Testing +```bash +# Set up local Mosquitto broker +sudo apt-get install mosquitto mosquitto-clients + +# Configure TLS (optional) +cat > /etc/mosquitto/conf.d/tls.conf << EOF +port 8883 +cafile /path/to/ca.crt +certfile /path/to/server.crt +keyfile /path/to/server.key +require_certificate true +use_identity_as_username true +EOF + +# Test local shadow operations +mosquitto_pub -h localhost -p 8883 \ + -t '$aws/things/test-device/shadow/update' \ + -m '{"state":{"reported":{"deviceState":"CONNECTED"}}}' +``` + +### Message Analysis and Documentation + +Expected shadow message patterns: + +```cpp +// Power state transitions +{ + "state": { + "reported": { + "deviceState": "CONNECTED", + "powerState": "ON|OFF" + } + }, + "timestamp": 1703875200 +} + +// Audio control updates +{ + "state": { + "reported": { + "volume": 25, + "muted": false, + "source": "SPOTIFY" + } + } +} + +// Multi-room coordination +{ + "state": { + "reported": { + "zoneState": "master|slave", + "groupMembers": ["device1", "device2"], + "groupName": "Living Room" + } + } +} +``` + +### Legal and Ethical Guidelines + +**Important Warnings:** +- Only monitor devices you personally own +- Using device credentials outside the device may violate Bose Terms of Service +- Accessing Bose's AWS infrastructure could be considered unauthorized +- Certificate abuse may result in device blacklisting +- Service shutdown in May 2026 makes this a temporary research opportunity + +**Recommended Usage:** +- Document message formats for local alternative development +- Understand state transition patterns +- Test compatibility with local MQTT brokers +- Prepare migration strategies before cloud shutdown + +### Research Implementation Example + +```cpp +class IoTResearchMonitor { +private: + std::string deviceClientId; + std::ofstream messageLog; + +public: + void captureMessagePatterns() { + // Subscribe only to owned device topics + std::string topic = "$aws/things/" + deviceClientId + "/shadow/update/accepted"; + + auto messageHandler = [this](const std::string& topic, const std::string& payload) { + // Log message structure for analysis + messageLog << "Topic: " << topic << std::endl; + messageLog << "Payload: " << payload << std::endl; + messageLog << "Timestamp: " << getCurrentTimestamp() << std::endl; + messageLog << "---" << std::endl; + + // Parse and document state transitions + documentStateTransition(payload); + }; + + // WARNING: Only use with your own device certificates + connectToAWSIoT(messageHandler); + } + + void documentStateTransition(const std::string& json) { + // Analyze JSON structure for local implementation + rapidjson::Document doc; + doc.Parse(json.c_str()); + + if (doc.HasMember("state") && doc["state"].HasMember("reported")) { + // Document field types and value ranges + auto& reported = doc["state"]["reported"]; + + for (auto& field : reported.GetObject()) { + std::cout << "Field: " << field.name.GetString() + << ", Type: " << getJSONType(field.value) << std::endl; + } + } + } +}; +``` + +This implementation guide provides the foundation for integrating with the Bose SoundTouch IoT system using AWS IoT Core, certificate-based authentication, and device shadow operations. The monitoring capabilities should be used responsibly and only for research purposes to develop local alternatives. \ No newline at end of file diff --git a/docs/guides/MIGRATION-GUIDE.md b/docs/guides/MIGRATION-GUIDE.md new file mode 100644 index 0000000..6fa4fbb --- /dev/null +++ b/docs/guides/MIGRATION-GUIDE.md @@ -0,0 +1,436 @@ +# THIS IS A PLANNED TO BE THE MIGRATION GUIDE + +> This migration guide is not finalized, yet. +> We're using it as an orientation for the required implementation. + +--- + +# Complete Migration Guide - From Bose Cloud to Local SoundTouch Service + +## Overview + +This guide will walk you through migrating your Bose SoundTouch speakers from Bose's cloud services to your own local SoundTouch service. By the end of this process, your speakers will be completely independent of Bose's servers while retaining all their functionality. + +> **πŸ’‘ Why Migrate?** Bose announced the shutdown of their SoundTouch cloud services in May 2026. This migration ensures your speakers continue working indefinitely with enhanced local control and monitoring. + +## What You'll Need + +### Hardware Requirements +- **Raspberry Pi 4 or similar** (minimum: Raspberry Pi Zero 2W) +- **MicroSD card** (16GB or larger) +- **USB drive** (for device preparation) +- **Network connection** for your Raspberry Pi + +### Before You Start +- **List all your SoundTouch devices** and their current locations +- **Note your current presets and favorites** (they will be preserved) +- **Ensure devices are on the same network** as your future SoundTouch service +- **Basic computer skills** (following instructions, using a web browser) + +### Time Estimate +- **Setup**: 30-60 minutes for the service installation +- **Per Device**: 10-15 minutes for each speaker migration +- **Total**: 1-3 hours depending on number of devices + +## Step 1: Install SoundTouch Service + +### Option A: Raspberry Pi Installation (Recommended) + +#### 1.1 Prepare Your Raspberry Pi + +1. **Flash Raspberry Pi OS** to your SD card using Raspberry Pi Imager (see the raspberrypi.com documentation) +2. **Enable SSH** during imaging or create an empty `ssh` file on the boot partition +3. **Boot your Pi** and connect it to your network +4. **Find your Pi's IP address** (check your router or use `ping raspberrypi.local`) + +#### 1.2 Install SoundTouch Service + +Connect to your Pi via SSH and run: + +```bash +# Download and install +curl -sSL https://github.com/gesellix/Bose-SoundTouch/releases/latest/download/install.sh | bash + +# Start the service +sudo systemctl enable soundtouch-service +sudo systemctl start soundtouch-service +``` + +#### 1.3 Verify Installation + +1. Open your web browser +2. Go to `http://[PI_IP_ADDRESS]:8000` (replace with your Pi's IP) +3. You should see the **SoundTouch Service Dashboard** + +![SoundTouch Service Dashboard](../images/dashboard-home.png) +*Example: SoundTouch Service main dashboard* + +### Option B: Docker Installation + +If you prefer Docker, run: + +```bash +docker run -d \ + --name soundtouch-service \ + --restart unless-stopped \ + -p 8000:8000 \ + -p 8443:8443 \ + -v soundtouch-data:/data \ + gesellix/soundtouch-service:latest +``` + +## Step 2: Create Your Account + +### 2.1 Initial Setup + +1. **Open the dashboard** at `http://[SERVICE_IP]:8000` +2. Click **"Create New Account"** +3. **Fill in your details**: + - Account Name: `My Home Audio` + - Email: `your@email.com` (optional, for notifications) + - Migration Strategy: `Gradual` (recommended) + +![Account Creation](../images/account-creation.png) +*Example: Account creation form* + +### 2.2 Account Configuration + +After creation, you'll see your **Account Dashboard**: +- **Account ID**: Unique identifier (e.g., `acc_home_audio_001`) +- **Status**: `Active - Ready for Migration` +- **Device Count**: Initially 0 +- **Migration Status**: `Prepared` + +![Account Dashboard](../images/account-dashboard.png) +*Example: Fresh account dashboard ready for device migration* + +## Step 3: Prepare Your Devices + +> **⚠️ Important**: This step temporarily enables SSH access on your speakers. SSH will be automatically disabled after migration unless you choose to keep it enabled. + +### 3.1 Enable Remote Services + +For each SoundTouch device: + +1. **Prepare a USB drive**: + - Format as FAT32 + - Create a file named `remote_services` (no extension) + - Content of file: `enable` + +2. **Insert USB drive** into your SoundTouch speaker +3. **Power cycle** the device (unplug for 10 seconds, then reconnect) + +![USB Preparation](../images/usb-remote-services.png) +*Example: USB drive setup for enabling remote services* + +### 3.2 Verify Remote Access + +In your SoundTouch Service dashboard: + +1. Go to **"Device Discovery"** tab +2. Click **"Scan Network"** +3. Your devices should appear with **SSH Status**: `Enabled` + +![Device Discovery](../images/device-discovery.png) +*Example: Discovered devices with remote access enabled* + +## Step 4: Discover and Register Devices + +### 4.1 Automatic Discovery + +The service automatically scans for SoundTouch devices every 5 minutes. To trigger immediate discovery: + +1. **Dashboard** β†’ **"Devices"** β†’ **"Discover Devices"** +2. **Wait 30-60 seconds** for scan completion +3. **Review discovered devices** in the list + +### 4.2 Register Devices to Your Account + +For each discovered device: + +1. **Click device name** in the discovery list +2. **Verify device information**: + - Name: `Living Room Speaker` + - Model: `SoundTouch 30` + - MAC Address: `A8:1B:6A:53:6A:98` + - IP Address: `192.168.1.100` + - Status: `Discovered - Ready for Registration` + +3. **Click "Register to Account"** +4. **Choose registration type**: + - **Fresh Setup**: For new or factory-reset devices + - **Migrate from Bose**: For devices with existing Bose account (recommended) + +![Device Registration](../images/device-registration.png) +*Example: Device registration dialog with migration options* + +### 4.3 Device Registration Results + +After registration, you'll see: +- **Device Status**: `Registered - Active` +- **Account Association**: Your account name +- **Lifecycle State**: `Active` +- **Data Sources**: `Mirror Primary` (initially uses Bose, falls back to local) + +## Step 5: Migrate Individual Devices + +> **πŸ”„ Migration Strategy**: We recommend "gradual migration" where devices continue using Bose services while building local data, then switch to local-only when ready. + +### 5.1 Start Device Migration + +1. **Dashboard** β†’ **"Devices"** β†’ Select your device +2. **Click "Migrate Device"** +3. **Configure migration**: + - Migration Method: `Gradual` (recommended) + - Preserve Data: βœ… `Yes` + - Backup Current State: βœ… `Yes` + - Test Period: `7 days` + +![Migration Setup](../images/migration-setup.png) +*Example: Migration configuration dialog* + +### 5.2 Migration Process + +The migration happens in phases: + +**Phase 1: Data Collection (Days 1-3)** +- Device continues using Bose services +- Local service mirrors all requests/responses +- Builds local database of presets, recents, sources +- Status: `Migrating - Data Collection` + +**Phase 2: Testing (Days 4-6)** +- Device uses local services for some requests +- Falls back to Bose if issues occur +- Validates local data completeness +- Status: `Migrating - Testing` + +**Phase 3: Full Local (Day 7+)** +- Device uses only local services +- Bose services disabled for this device +- Full independence achieved +- Status: `Active - Local Only` + +![Migration Progress](../images/migration-progress.png) +*Example: Migration progress tracking* + +### 5.3 Monitor Migration Health + +During migration, monitor: + +**Device Health Dashboard**: +- **Connectivity**: Should remain `Online` throughout +- **Response Time**: Should be similar or better than before +- **Error Rate**: Should remain low (<1%) +- **Data Completeness**: Shows percentage of data successfully migrated + +![Migration Health](../images/migration-health.png) +*Example: Migration health monitoring* + +**Data Verification**: +- **Presets**: Verify all presets work correctly +- **Recent History**: Check recent play history is preserved +- **Source Configuration**: Ensure all configured sources (Spotify, etc.) work +- **Multiroom**: Test zone functionality if applicable + +## Step 6: Complete Account Migration + +### 6.1 Migrate All Devices + +Repeat Step 5 for each of your SoundTouch devices. You can migrate multiple devices simultaneously, but we recommend doing 1-2 at a time to monitor progress. + +**Migration Dashboard** shows overall progress: +- **Devices Migrated**: `2 of 4 completed` +- **Currently Migrating**: `Living Room Speaker, Kitchen Speaker` +- **Pending Migration**: `Bedroom Speaker, Office Speaker` +- **Estimated Completion**: `3 days remaining` + +![Account Migration Status](../images/account-migration.png) +*Example: Account-wide migration progress* + +### 6.2 Verify Complete Migration + +When all devices are migrated: + +1. **Account Status**: `Active - Fully Migrated` +2. **Bose Dependency**: `None` +3. **Local Control**: `100%` +4. **Device Health**: All devices show `Healthy - Local Only` + +![Migration Complete](../images/migration-complete.png) +*Example: Completed migration dashboard* + +## Step 7: Post-Migration Setup + +### 7.1 Disable Remote Services (Optional) + +For enhanced security, disable SSH on migrated devices: + +1. **Dashboard** β†’ **"Device Management"** +2. **Select device** β†’ **"Security Settings"** +3. **Click "Disable Remote Services"** +4. **Confirm** to disable SSH access + +### 7.2 Configure Backups + +Set up automatic backups of your device configurations: + +1. **Dashboard** β†’ **"Settings"** β†’ **"Backup"** +2. **Enable Automatic Backups**: βœ… +3. **Backup Schedule**: `Daily at 2 AM` +4. **Retention**: `Keep 30 days` +5. **Export Location**: `/data/backups` or external storage + +![Backup Configuration](../images/backup-setup.png) +*Example: Backup configuration settings* + +### 7.3 Set Up Monitoring Alerts (Optional) + +Configure notifications for important events: + +1. **Dashboard** β†’ **"Settings"** β†’ **"Notifications"** +2. **Email Notifications**: Enter your email +3. **Alert Types**: + - βœ… Device goes offline + - βœ… Migration failures + - βœ… Service errors + - βœ… Daily health summary + +## Troubleshooting Common Issues + +### Device Not Discovered + +**Problem**: Device doesn't appear in discovery scan + +**Solutions**: +1. **Check network**: Ensure device and service are on same network +2. **Verify USB setup**: Confirm `remote_services` file was processed +3. **Power cycle**: Unplug device for 30 seconds, reconnect +4. **Manual add**: Dashboard β†’ "Devices" β†’ "Add Manually" with IP address + +### Migration Stuck + +**Problem**: Device stuck in "Migrating" status + +**Solutions**: +1. **Check device health**: Dashboard β†’ Device β†’ "Health Status" +2. **Review logs**: Dashboard β†’ Device β†’ "View Logs" +3. **Restart migration**: Device β†’ "Migration" β†’ "Restart Process" +4. **Rollback**: Device β†’ "Migration" β†’ "Rollback to Bose" + +### Presets Not Working + +**Problem**: Saved presets don't work after migration + +**Solutions**: +1. **Verify sources**: Check configured sources are still available +2. **Re-authenticate**: Re-login to music services (Spotify, etc.) +3. **Rebuild presets**: Dashboard β†’ Device β†’ "Presets" β†’ "Rebuild from Backup" + +### Service Unreachable + +**Problem**: Cannot access SoundTouch Service dashboard + +**Solutions**: +1. **Check service status**: `sudo systemctl status soundtouch-service` +2. **Restart service**: `sudo systemctl restart soundtouch-service` +3. **Check network**: Verify Pi is connected and accessible +4. **Check ports**: Ensure ports 8000 and 8443 are not blocked + +## Advanced Features + +### Multi-Zone Management + +After migration, your multi-zone setups work seamlessly: + +1. **Dashboard** β†’ **"Zones"** +2. **Create Zone**: Select primary device and slaves +3. **Zone Control**: Play, pause, volume control for entire zone +4. **Individual Control**: Override individual speakers in zone + +### Custom Sources + +Add custom streaming sources: + +1. **Dashboard** β†’ **"Sources"** β†’ **"Add Custom"** +2. **Configure**: + - Name: `Local Radio Station` + - Stream URL: `http://stream.example.com:8000` + - Image URL: `http://example.com/logo.png` +3. **Assign to devices**: Select which devices can access this source + +### API Access + +For developers and advanced users: + +- **REST API**: `http://[SERVICE_IP]:8000/api/v1/` +- **Documentation**: `http://[SERVICE_IP]:8000/docs` +- **WebSocket Events**: Real-time device status updates +- **Export Data**: JSON/XML export of all device configurations + +## Maintenance and Monitoring + +### Daily Monitoring + +Check your **Dashboard Summary**: +- **All Devices Online**: βœ… Green indicators +- **Response Times**: < 100ms average +- **Error Rate**: < 1% +- **Storage Usage**: Monitor disk space + +### Weekly Tasks + +1. **Review Health Reports**: Check weekly device health summaries +2. **Update Service**: Check for SoundTouch service updates +3. **Backup Verification**: Ensure backups are completing successfully +4. **Log Review**: Check for any recurring issues or warnings + +### Monthly Tasks + +1. **Full System Backup**: Export complete account and device data +2. **Performance Review**: Analyze response times and error patterns +3. **Security Update**: Update Raspberry Pi OS and service +4. **Capacity Planning**: Monitor storage and consider expansion + +## Getting Help + +### Documentation Resources + +- **Technical Reference**: `/docs/reference/` - Detailed API and configuration docs +- **Troubleshooting Guide**: `/docs/guides/TROUBLESHOOTING.md` - Common issues and solutions +- **Community Forum**: GitHub Discussions for community support + +### Diagnostic Information + +When seeking help, provide: + +1. **System Information**: Dashboard β†’ "System" β†’ "Download Diagnostic Report" +2. **Device Logs**: Dashboard β†’ Device β†’ "Export Logs" +3. **Migration History**: Dashboard β†’ "Migration" β†’ "Export Timeline" +4. **Current Status**: Screenshot of main dashboard + +### Support Channels + +- **GitHub Issues**: Technical bugs and feature requests +- **Community Discussions**: User questions and experiences +- **Documentation Updates**: Corrections and improvements + +--- + +## Summary + +Congratulations! πŸŽ‰ You've successfully migrated your SoundTouch speakers to local control. Your devices are now: + +- βœ… **Independent** of Bose cloud services +- βœ… **Fully functional** with all original features preserved +- βœ… **Enhanced** with better monitoring and control +- βœ… **Future-proof** against service shutdowns + +**What's Next?** + +- **Enjoy your music** with enhanced local control +- **Monitor your system** through the dashboard +- **Share your experience** with the community +- **Explore advanced features** as you become more comfortable + +Your SoundTouch speakers will now continue working indefinitely, regardless of external service availability. Welcome to true audio independence! πŸ”Š diff --git a/docs/guides/MQTT-INTEGRATION-DESIGN.md b/docs/guides/MQTT-INTEGRATION-DESIGN.md new file mode 100644 index 0000000..3f220aa --- /dev/null +++ b/docs/guides/MQTT-INTEGRATION-DESIGN.md @@ -0,0 +1,764 @@ +# MQTT Integration Design for SoundTouch Service + +## Overview + +This document outlines the design for integrating MQTT support into the existing SoundTouch service to simulate AWS IoT Core functionality. The integration will provide real-time device communication, shadow state management, and prepare for the AWS IoT service shutdown in May 2026. + +## Current Architecture Analysis + +### Existing Service Structure +``` +Bose-SoundTouch/ +β”œβ”€β”€ cmd/soundtouch-service/main.go # Main service entry point +β”œβ”€β”€ pkg/ +β”‚ β”œβ”€β”€ client/ # HTTP client for devices +β”‚ β”œβ”€β”€ config/ # Configuration management +β”‚ β”œβ”€β”€ discovery/ # Device discovery (UPnP, mDNS) +β”‚ β”œβ”€β”€ models/ # Data structures +β”‚ └── service/ +β”‚ β”œβ”€β”€ handlers/ # HTTP request handlers +β”‚ β”‚ └── server.go # Main server struct +β”‚ β”œβ”€β”€ datastore/ # Data persistence +β”‚ β”œβ”€β”€ proxy/ # HTTP proxying +β”‚ └── [other services] +``` + +### Key Components +- **Server Struct**: Central HTTP handler in `pkg/service/handlers/server.go` +- **Discovery Service**: UPnP/mDNS device discovery in `pkg/discovery/` +- **DataStore**: Device state persistence in `pkg/service/datastore/` +- **Device Models**: Data structures in `pkg/models/` + +## MQTT Integration Design + +### 1. New Package Structure + +``` +pkg/service/mqtt/ +β”œβ”€β”€ broker.go # MQTT broker implementation +β”œβ”€β”€ shadow.go # AWS IoT Shadow simulation +β”œβ”€β”€ auth.go # Certificate-based authentication +β”œβ”€β”€ topics.go # Topic routing and handlers +β”œβ”€β”€ bridge.go # HTTP ↔ MQTT state bridging +β”œβ”€β”€ config.go # MQTT configuration +└── client.go # MQTT client utilities +``` + +### 2. Core Components + +#### A. MQTT Broker (`pkg/service/mqtt/broker.go`) +```go +package mqtt + +import ( + "crypto/tls" + "fmt" + "log" + "sync" + + "github.com/mochi-co/mqtt/v2" + "github.com/mochi-co/mqtt/v2/hooks/auth" + "github.com/mochi-co/mqtt/v2/listeners" +) + +type Broker struct { + server *mqtt.Server + shadowStore *ShadowStore + bridge *HTTPBridge + authHook *AuthHook + config *Config + running bool + mu sync.RWMutex +} + +type Config struct { + Enabled bool `json:"enabled"` + Port int `json:"port"` + TLSEnabled bool `json:"tls_enabled"` + CertFile string `json:"cert_file"` + KeyFile string `json:"key_file"` + DeviceCertPath string `json:"device_cert_path"` + ShadowPersist bool `json:"shadow_persist"` +} + +func NewBroker(config *Config) (*Broker, error) { + server := mqtt.New(nil) + + shadowStore := NewShadowStore() + authHook := NewAuthHook(config.DeviceCertPath) + + return &Broker{ + server: server, + shadowStore: shadowStore, + authHook: authHook, + config: config, + }, nil +} + +func (b *Broker) Start() error { + // Add TLS listener + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{b.loadServerCert()}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: b.loadDeviceCAs(), + } + + tcp := listeners.NewTCP("mqtt-tls", fmt.Sprintf(":%d", b.config.Port), &listeners.Config{ + TLSConfig: tlsConfig, + }) + + b.server.AddListener(tcp) + + // Add hooks + b.server.AddHook(b.authHook, nil) + b.server.AddHook(NewShadowHook(b.shadowStore), nil) + + return b.server.Serve() +} +``` + +#### B. Shadow State Management (`pkg/service/mqtt/shadow.go`) +```go +package mqtt + +import ( + "encoding/json" + "fmt" + "sync" + "time" +) + +type ShadowStore struct { + shadows map[string]*DeviceShadow + mu sync.RWMutex +} + +type DeviceShadow struct { + State struct { + Desired map[string]interface{} `json:"desired"` + Reported map[string]interface{} `json:"reported"` + Delta map[string]interface{} `json:"delta,omitempty"` + } `json:"state"` + Version int `json:"version"` + Timestamp int64 `json:"timestamp"` + ClientToken string `json:"clientToken,omitempty"` +} + +func NewShadowStore() *ShadowStore { + return &ShadowStore{ + shadows: make(map[string]*DeviceShadow), + } +} + +func (s *ShadowStore) UpdateShadow(clientID string, payload []byte) (*DeviceShadow, error) { + s.mu.Lock() + defer s.mu.Unlock() + + var update DeviceShadow + if err := json.Unmarshal(payload, &update); err != nil { + return nil, err + } + + shadow := s.shadows[clientID] + if shadow == nil { + shadow = &DeviceShadow{ + State: struct { + Desired map[string]interface{} `json:"desired"` + Reported map[string]interface{} `json:"reported"` + Delta map[string]interface{} `json:"delta,omitempty"` + }{ + Desired: make(map[string]interface{}), + Reported: make(map[string]interface{}), + Delta: make(map[string]interface{}), + }, + } + s.shadows[clientID] = shadow + } + + // Update reported state + if update.State.Reported != nil { + for key, value := range update.State.Reported { + shadow.State.Reported[key] = value + } + } + + // Update desired state + if update.State.Desired != nil { + for key, value := range update.State.Desired { + shadow.State.Desired[key] = value + } + } + + // Calculate delta + shadow.calculateDelta() + shadow.Version++ + shadow.Timestamp = time.Now().Unix() + shadow.ClientToken = update.ClientToken + + return shadow, nil +} + +func (s *DeviceShadow) calculateDelta() { + s.State.Delta = make(map[string]interface{}) + + for key, desired := range s.State.Desired { + if reported, exists := s.State.Reported[key]; !exists || reported != desired { + s.State.Delta[key] = desired + } + } + + if len(s.State.Delta) == 0 { + s.State.Delta = nil + } +} +``` + +#### C. HTTP ↔ MQTT Bridge (`pkg/service/mqtt/bridge.go`) +```go +package mqtt + +import ( + "encoding/json" + "fmt" + "log" + + "github.com/gesellix/bose-soundtouch/pkg/models" + "github.com/gesellix/bose-soundtouch/pkg/service/datastore" +) + +type HTTPBridge struct { + shadowStore *ShadowStore + dataStore *datastore.DataStore + deviceMap map[string]string // clientID -> deviceID mapping +} + +func NewHTTPBridge(shadowStore *ShadowStore, dataStore *datastore.DataStore) *HTTPBridge { + return &HTTPBridge{ + shadowStore: shadowStore, + dataStore: dataStore, + deviceMap: make(map[string]string), + } +} + +// ShadowToHTTP converts MQTT shadow updates to HTTP API calls +func (b *HTTPBridge) ShadowToHTTP(clientID string, shadow *DeviceShadow) error { + deviceID, exists := b.deviceMap[clientID] + if !exists { + log.Printf("Unknown device clientID: %s", clientID) + return fmt.Errorf("unknown device: %s", clientID) + } + + // Handle power state changes + if powerState, ok := shadow.State.Reported["powerState"].(string); ok { + if err := b.updateDevicePower(deviceID, powerState == "ON"); err != nil { + return fmt.Errorf("power update failed: %w", err) + } + } + + // Handle volume changes + if volume, ok := shadow.State.Reported["volume"].(float64); ok { + if err := b.updateDeviceVolume(deviceID, int(volume)); err != nil { + return fmt.Errorf("volume update failed: %w", err) + } + } + + // Handle source changes + if source, ok := shadow.State.Reported["source"].(string); ok { + if err := b.updateDeviceSource(deviceID, source); err != nil { + return fmt.Errorf("source update failed: %w", err) + } + } + + return nil +} + +// HTTPToShadow converts HTTP device state to MQTT shadow updates +func (b *HTTPBridge) HTTPToShadow(deviceID string, deviceInfo *models.DeviceInfo) error { + clientID, exists := b.getClientIDForDevice(deviceID) + if !exists { + return nil // Device not connected via MQTT + } + + // Create shadow state from device info + shadowState := map[string]interface{}{ + "deviceState": "CONNECTED", + "deviceID": deviceInfo.DeviceID, + "name": deviceInfo.Name, + "type": deviceInfo.Type, + } + + // Add additional state if available + if status := b.getDeviceStatus(deviceID); status != nil { + shadowState["powerState"] = status.PowerState + shadowState["volume"] = status.Volume + shadowState["source"] = status.Source + } + + // Update shadow + shadowUpdate := DeviceShadow{ + State: struct { + Desired map[string]interface{} `json:"desired"` + Reported map[string]interface{} `json:"reported"` + Delta map[string]interface{} `json:"delta,omitempty"` + }{ + Reported: shadowState, + }, + } + + payload, _ := json.Marshal(shadowUpdate) + _, err := b.shadowStore.UpdateShadow(clientID, payload) + return err +} +``` + +### 3. Integration with Existing Server + +#### A. Extend Server Struct (`pkg/service/handlers/server.go`) +```go +// Add to existing Server struct +type Server struct { + // ... existing fields ... + + // New MQTT fields + mqttBroker *mqtt.Broker + mqttEnabled bool + mqttConfig *mqtt.Config + deviceClientIDs map[string]string // deviceID -> clientID mapping +} + +// New initialization method +func (s *Server) initMQTTBroker(config *mqtt.Config) error { + if !config.Enabled { + return nil + } + + broker, err := mqtt.NewBroker(config) + if err != nil { + return fmt.Errorf("failed to create MQTT broker: %w", err) + } + + // Set up HTTP ↔ MQTT bridge + bridge := mqtt.NewHTTPBridge(broker.ShadowStore(), s.ds) + broker.SetBridge(bridge) + + s.mqttBroker = broker + s.mqttEnabled = true + s.mqttConfig = config + s.deviceClientIDs = make(map[string]string) + + return nil +} + +// Start MQTT broker alongside HTTP server +func (s *Server) StartMQTT() error { + if !s.mqttEnabled { + return nil + } + + go func() { + if err := s.mqttBroker.Start(); err != nil { + log.Printf("MQTT broker error: %v", err) + } + }() + + return nil +} +``` + +#### B. Configuration Integration (`cmd/soundtouch-service/main.go`) +```go +// Add to serviceConfig struct +type serviceConfig struct { + // ... existing fields ... + + // New MQTT configuration fields + mqttEnabled bool `mapstructure:"mqtt_enabled"` + mqttPort int `mapstructure:"mqtt_port"` + mqttTLSCert string `mapstructure:"mqtt_tls_cert"` + mqttTLSKey string `mapstructure:"mqtt_tls_key"` + mqttDeviceCertPath string `mapstructure:"mqtt_device_cert_path"` + mqttShadowPersist bool `mapstructure:"mqtt_shadow_persist"` +} + +// Update main function to initialize MQTT +func main() { + // ... existing initialization ... + + // Initialize MQTT if enabled + if cfg.mqttEnabled { + mqttConfig := &mqtt.Config{ + Enabled: cfg.mqttEnabled, + Port: cfg.mqttPort, + TLSEnabled: true, + CertFile: cfg.mqttTLSCert, + KeyFile: cfg.mqttTLSKey, + DeviceCertPath: cfg.mqttDeviceCertPath, + ShadowPersist: cfg.mqttShadowPersist, + } + + if err := server.InitMQTTBroker(mqttConfig); err != nil { + log.Fatalf("Failed to initialize MQTT broker: %v", err) + } + + if err := server.StartMQTT(); err != nil { + log.Fatalf("Failed to start MQTT broker: %v", err) + } + + log.Printf("MQTT broker started on port %d", cfg.mqttPort) + } + + // ... rest of existing main function ... +} +``` + +### 4. Enhanced Device Discovery + +#### A. MQTT Device Discovery (`pkg/service/mqtt/discovery.go`) +```go +package mqtt + +import ( + "log" + "time" + + "github.com/gesellix/bose-soundtouch/pkg/models" + "github.com/mochi-co/mqtt/v2/packets" +) + +type DeviceDiscoveryHook struct { + deviceRegistry map[string]*models.Device + onDeviceFound func(*models.Device) +} + +func NewDeviceDiscoveryHook() *DeviceDiscoveryHook { + return &DeviceDiscoveryHook{ + deviceRegistry: make(map[string]*models.Device), + } +} + +func (h *DeviceDiscoveryHook) ID() string { + return "device-discovery" +} + +func (h *DeviceDiscoveryHook) OnConnect(cl *packets.Client, pk packets.Packet) error { + clientID := pk.Connect.ClientIdentifier + + log.Printf("MQTT device connected: %s", clientID) + + // Create device entry + device := &models.Device{ + ID: clientID, + ClientID: clientID, + Name: "MQTT Device", + LastSeen: time.Now(), + MQTTOnline: true, + Source: "mqtt", + } + + h.deviceRegistry[clientID] = device + + if h.onDeviceFound != nil { + h.onDeviceFound(device) + } + + return nil +} + +func (h *DeviceDiscoveryHook) OnDisconnect(cl *packets.Client, err error) { + clientID := cl.ID + + log.Printf("MQTT device disconnected: %s", clientID) + + if device, exists := h.deviceRegistry[clientID]; exists { + device.MQTTOnline = false + device.LastSeen = time.Now() + } +} +``` + +#### B. Integration with Existing Discovery (`pkg/discovery/mqtt.go`) +```go +package discovery + +import ( + "context" + "time" + + "github.com/gesellix/bose-soundtouch/pkg/models" +) + +type MQTTDiscovery struct { + deviceRegistry map[string]*models.Device + enabled bool +} + +func NewMQTTDiscovery() *MQTTDiscovery { + return &MQTTDiscovery{ + deviceRegistry: make(map[string]*models.Device), + enabled: true, + } +} + +func (d *MQTTDiscovery) DiscoverDevices(ctx context.Context, timeout time.Duration) ([]*models.Device, error) { + if !d.enabled { + return []*models.Device{}, nil + } + + var devices []*models.Device + for _, device := range d.deviceRegistry { + if device.MQTTOnline { + devices = append(devices, device) + } + } + + return devices, nil +} + +func (d *MQTTDiscovery) AddDevice(device *models.Device) { + d.deviceRegistry[device.ClientID] = device +} + +func (d *MQTTDiscovery) RemoveDevice(clientID string) { + delete(d.deviceRegistry, clientID) +} +``` + +### 5. Configuration File Extensions + +#### A. Default Configuration (`config.yaml`) +```yaml +# Existing configuration... + +# MQTT Configuration +mqtt: + enabled: false + port: 8883 + tls: + cert_file: "/etc/ssl/certs/soundtouch-mqtt.crt" + key_file: "/etc/ssl/private/soundtouch-mqtt.key" + + # Device certificate validation + device_certs: + path: "/etc/soundtouch/device-certs" + auto_load: true + + # Shadow state management + shadow: + persist: true + ttl: 86400 # 24 hours + + # Bridge configuration + bridge: + enabled: true + sync_interval: 30s +``` + +#### B. Environment Variable Support +```bash +# MQTT configuration via environment variables +SOUNDTOUCH_MQTT_ENABLED=true +SOUNDTOUCH_MQTT_PORT=8883 +SOUNDTOUCH_MQTT_TLS_CERT=/path/to/cert.pem +SOUNDTOUCH_MQTT_TLS_KEY=/path/to/key.pem +SOUNDTOUCH_MQTT_DEVICE_CERT_PATH=/path/to/device/certs +SOUNDTOUCH_MQTT_SHADOW_PERSIST=true +``` + +### 6. API Extensions + +#### A. MQTT Status Endpoints +```go +// Add to handlers +func (s *Server) handleMQTTStatus(c *gin.Context) { + if !s.mqttEnabled { + c.JSON(http.StatusNotImplemented, gin.H{ + "error": "MQTT not enabled", + }) + return + } + + status := gin.H{ + "enabled": s.mqttEnabled, + "port": s.mqttConfig.Port, + "connected_devices": len(s.deviceClientIDs), + "shadow_count": s.mqttBroker.ShadowStore().Count(), + } + + c.JSON(http.StatusOK, status) +} + +// Device shadow endpoint +func (s *Server) handleDeviceShadow(c *gin.Context) { + deviceID := c.Param("deviceId") + clientID, exists := s.deviceClientIDs[deviceID] + if !exists { + c.JSON(http.StatusNotFound, gin.H{ + "error": "Device not connected via MQTT", + }) + return + } + + shadow := s.mqttBroker.ShadowStore().GetShadow(clientID) + if shadow == nil { + c.JSON(http.StatusNotFound, gin.H{ + "error": "Shadow not found", + }) + return + } + + c.JSON(http.StatusOK, shadow) +} +``` + +### 7. Testing Strategy + +#### A. Unit Tests +```go +// pkg/service/mqtt/shadow_test.go +func TestShadowStore_UpdateShadow(t *testing.T) { + store := NewShadowStore() + + payload := []byte(`{ + "state": { + "reported": { + "powerState": "ON", + "volume": 25 + } + } + }`) + + shadow, err := store.UpdateShadow("test-client", payload) + assert.NoError(t, err) + assert.Equal(t, "ON", shadow.State.Reported["powerState"]) + assert.Equal(t, 25.0, shadow.State.Reported["volume"]) + assert.Equal(t, 1, shadow.Version) +} +``` + +#### B. Integration Tests +```go +// pkg/service/mqtt/integration_test.go +func TestMQTTBrokerIntegration(t *testing.T) { + // Start test broker + broker := setupTestBroker(t) + go broker.Start() + defer broker.Stop() + + // Connect test client + client := mqtt.NewClient(mqtt.NewClientOptions(). + AddBroker("tls://localhost:8883"). + SetClientID("test-device")) + + // Test shadow operations + testShadowUpdate(t, client) + testShadowGet(t, client) +} +``` + +### 8. Migration Path + +#### A. Gradual Rollout +1. **Phase 1**: Deploy MQTT broker alongside existing HTTP service (disabled by default) +2. **Phase 2**: Enable MQTT for testing with specific devices +3. **Phase 3**: Enable bidirectional HTTP ↔ MQTT bridging +4. **Phase 4**: Full MQTT support for all discovered devices +5. **Phase 5**: Prepare for AWS IoT shutdown (May 2026) + +#### B. Backward Compatibility +- All existing HTTP API endpoints continue to work +- MQTT is purely additive functionality +- Devices can be discovered via HTTP even with MQTT enabled +- Configuration remains optional + +### 9. Monitoring and Logging + +#### A. MQTT Metrics +```go +type MQTTMetrics struct { + ConnectedDevices int64 + MessagesReceived int64 + MessagesSent int64 + ShadowUpdates int64 + AuthenticationFails int64 + Uptime time.Duration +} + +func (b *Broker) GetMetrics() *MQTTMetrics { + return &MQTTMetrics{ + ConnectedDevices: int64(len(b.server.Clients)), + MessagesReceived: b.server.Stats.MessagesReceived, + MessagesSent: b.server.Stats.MessagesSent, + ShadowUpdates: b.shadowStore.UpdateCount(), + AuthenticationFails: b.authHook.FailCount(), + Uptime: time.Since(b.startTime), + } +} +``` + +#### B. Logging Integration +```go +import "github.com/sirupsen/logrus" + +func (b *Broker) setupLogging() { + log := logrus.WithFields(logrus.Fields{ + "component": "mqtt-broker", + "port": b.config.Port, + }) + + b.server.AddHook(&LoggingHook{logger: log}, nil) +} +``` + +### 10. Security Considerations + +#### A. Certificate Validation +- Validate device certificates against known device list +- Implement certificate revocation checking +- Support certificate rotation + +#### B. Access Control +- Restrict topic access per device certificate +- Implement rate limiting per client +- Monitor for unusual connection patterns + +#### C. Data Protection +- Encrypt shadow data at rest +- Implement secure certificate storage +- Audit logging for security events + +## Implementation Timeline + +### Week 1: Core Infrastructure +- [ ] Create MQTT package structure +- [ ] Implement basic MQTT broker +- [ ] Add TLS configuration +- [ ] Basic shadow state management + +### Week 2: Integration & Bridging +- [ ] Integrate with existing Server struct +- [ ] Implement HTTP ↔ MQTT bridge +- [ ] Device discovery integration +- [ ] Configuration management + +### Week 3: Testing & Polish +- [ ] Unit test coverage +- [ ] Integration testing +- [ ] Documentation updates +- [ ] Performance optimization + +### Week 4: Deployment & Monitoring +- [ ] Docker container updates +- [ ] Monitoring and metrics +- [ ] Security hardening +- [ ] Production readiness + +## Success Criteria + +1. **Functional**: MQTT broker accepts device connections using extracted certificates +2. **Compatible**: All existing HTTP functionality continues to work unchanged +3. **Performant**: MQTT operations don't impact HTTP API performance +4. **Secure**: Device authentication and authorization properly implemented +5. **Observable**: Comprehensive logging and metrics for MQTT operations +6. **Maintainable**: Clean separation of MQTT code from existing HTTP logic + +This design provides a comprehensive path to add MQTT support while maintaining the existing architecture and ensuring smooth integration with current functionality. \ No newline at end of file diff --git a/docs/images/README.md b/docs/images/README.md new file mode 100644 index 0000000..f50e49d --- /dev/null +++ b/docs/images/README.md @@ -0,0 +1,90 @@ +# Images for Migration Guide + +This directory contains images, screenshots, and diagrams referenced in the migration guide and other documentation. + +## Required Images for Migration Guide + +The following images need to be created to complete the migration guide: + +### Dashboard Screenshots +- **dashboard-home.png** - Main SoundTouch Service dashboard homepage +- **account-creation.png** - Account creation form with fields filled +- **account-dashboard.png** - Fresh account dashboard showing ready state +- **device-discovery.png** - Device discovery page showing found speakers +- **device-registration.png** - Device registration dialog with options +- **migration-setup.png** - Migration configuration dialog +- **migration-progress.png** - Migration progress tracker showing phases +- **migration-health.png** - Migration health monitoring dashboard +- **account-migration.png** - Account-wide migration progress overview +- **migration-complete.png** - Completed migration dashboard view +- **backup-setup.png** - Backup configuration settings page + +### Setup and Preparation +- **usb-remote-services.png** - USB drive setup showing file structure +- **raspberry-pi-setup.png** - Raspberry Pi with connected cables (optional) + +### Process Diagrams +- **migration-flow-diagram.png** - Flow chart showing migration phases +- **network-topology.png** - Network diagram showing Pi, router, speakers +- **data-flow-diagram.png** - How data flows between components + +## Image Requirements + +### Technical Specifications +- **Format**: PNG preferred for screenshots, SVG for diagrams +- **Resolution**: Minimum 1200px width for screenshots +- **File Size**: Keep under 500KB when possible for fast loading +- **Naming**: Use descriptive kebab-case names as shown above + +### Content Guidelines +- **Clean Interface**: Show realistic but clean interface states +- **Consistent Styling**: Use consistent colors and styling across images +- **Readable Text**: Ensure all text in screenshots is legible +- **Example Data**: Use realistic example data (Living Room Speaker, etc.) +- **Status Indicators**: Show clear success/error states with appropriate colors + +### Placeholder Content +Until real screenshots are available, consider: +- **Mockups**: Create simple mockups showing the expected interface +- **Wireframes**: Basic wireframes indicating layout and content +- **Diagrams**: Technical diagrams can be created immediately +- **Text Placeholders**: Use `[Image: Description]` in documentation + +## Creating the Images + +### For Dashboard Screenshots +1. Set up the enhanced SoundTouch service +2. Create sample account and register devices +3. Take screenshots at key points in the migration process +4. Edit for clarity (highlight important elements, add annotations) + +### For Diagrams +1. Use tools like Lucidchart, draw.io, or similar +2. Follow consistent color scheme: + - Blue: SoundTouch Service components + - Green: Healthy/successful states + - Orange: Warning/in-progress states + - Red: Error/problematic states + - Gray: External/third-party components + +### For Physical Setup +1. Take photos of actual hardware setup +2. Show USB drive preparation process +3. Demonstrate network connections if helpful + +## Alternative Text Requirements + +Each image should have appropriate alt text for accessibility: + +```markdown +![Alt text describing the image content](../images/image-name.png) +*Caption: Additional context or explanation* +``` + +## Future Enhancements + +Consider adding: +- **Video Walkthroughs**: Screen recordings of key processes +- **Interactive Demos**: Web-based interactive guides +- **Troubleshooting Screenshots**: Common error states and solutions +- **Mobile Views**: How to access from mobile devices \ No newline at end of file diff --git a/docs/power-on-implementation-guide.md b/docs/power-on-implementation-guide.md new file mode 100644 index 0000000..ad016fb --- /dev/null +++ b/docs/power-on-implementation-guide.md @@ -0,0 +1,599 @@ +# /power_on Implementation Guide + +## Overview + +This guide provides detailed technical specifications for implementing `/power_on` endpoint enhancements to reduce network dependency and improve device lifecycle management in the SoundTouch service. + +## Current /power_on Handler Analysis + +### Existing Implementation +Located in `pkg/service/handlers/handlers_marge.go`: + +```go +func (s *Server) HandleMargePowerOn(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + log.Printf("[Marge] Failed to read power_on body: %v", err) + w.WriteHeader(http.StatusOK) + return + } + + var req models.CustomerSupportRequest + if err := xml.Unmarshal(body, &req); err != nil { + log.Printf("[Marge] Failed to parse power_on body: %v", err) + // Fallback to remote address + if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil { + go s.PrimeDeviceWithSpotify(host) + } + w.WriteHeader(http.StatusOK) + return + } + + deviceID := req.Device.ID + deviceIP := req.DiagnosticData.DeviceLandscape.IPAddress + + log.Printf("[Marge] Device %s powered on (IP: %s)", deviceID, deviceIP) + + if deviceIP != "" { + go s.PrimeDeviceWithSpotify(deviceIP) + } else { + // Fallback to remote address + if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil { + go s.PrimeDeviceWithSpotify(host) + } + } + + w.WriteHeader(http.StatusOK) +} +``` + +**Current Limitations:** +- Only extracts basic device ID and IP +- No device state management +- No data persistence +- No response payload +- Limited to Spotify priming + +## Enhanced Implementation Design + +### 1. Extended Data Models + +#### Enhanced Power-On Request Model +```go +// PowerOnRequest represents the enhanced power_on request structure +type PowerOnRequest struct { + XMLName xml.Name `xml:"device-data"` + Device PowerOnDevice `xml:"device"` + DiagnosticData DiagnosticData `xml:"diagnostic-data"` +} + +type PowerOnDevice struct { + ID string `xml:"id,attr"` + SerialNumber string `xml:"serialnumber"` + FirmwareVersion string `xml:"firmware-version"` + Product PowerOnProduct `xml:"product"` +} + +type PowerOnProduct struct { + ProductCode string `xml:"product_code,attr"` + Type string `xml:"type,attr"` + SerialNumber string `xml:"serialnumber"` +} + +type DiagnosticData struct { + DeviceLandscape DeviceLandscape `xml:"device-landscape"` + NetworkData NetworkData `xml:"network-landscape>network-data"` +} + +type DeviceLandscape struct { + RSSI string `xml:"rssi"` + GatewayIP string `xml:"gateway-ip-address"` + MacAddresses []string `xml:"macaddresses>macaddress"` + IPAddress string `xml:"ip-address"` + ConnectionType string `xml:"network-connection-type"` +} +``` + +#### Enhanced Response Model +```go +// PowerOnResponse represents the response sent back to the device +type PowerOnResponse struct { + XMLName xml.Name `xml:"power-on-response"` + Status string `xml:"status"` + DeviceID string `xml:"device-id"` + ConfigurationUpdates []ConfigurationUpdate `xml:"configuration-updates>update,omitempty"` + MigrationInstructions *MigrationInstruction `xml:"migration,omitempty"` + RegistrationRequired bool `xml:"registration-required,omitempty"` + Timestamp string `xml:"timestamp"` +} + +type ConfigurationUpdate struct { + Type string `xml:"type,attr"` + Key string `xml:"key"` + Value string `xml:"value"` + Priority int `xml:"priority,attr"` +} + +type MigrationInstruction struct { + Method string `xml:"method,attr"` + TargetURL string `xml:"target-url"` + ProxyURL string `xml:"proxy-url,omitempty"` + Options map[string]string `xml:"options>option"` +} +``` + +### 2. Enhanced PowerOn Handler + +```go +// HandleMargePowerOnEnhanced processes power_on requests with full device lifecycle management +func (s *Server) HandleMargePowerOnEnhanced(w http.ResponseWriter, r *http.Request) { + startTime := time.Now() + + // Parse the power_on request + powerOnReq, err := s.parsePowerOnRequest(r) + if err != nil { + s.handlePowerOnError(w, r, "Failed to parse request", err) + return + } + + // Process device information + deviceInfo, isNewDevice, err := s.processDeviceFromPowerOn(powerOnReq) + if err != nil { + s.handlePowerOnError(w, r, "Failed to process device", err) + return + } + + // Build response based on device state + response := s.buildPowerOnResponse(deviceInfo, isNewDevice, powerOnReq) + + // Log the interaction + s.logPowerOnInteraction(deviceInfo, powerOnReq, response, startTime) + + // Send response + if err := s.sendPowerOnResponse(w, response); err != nil { + log.Printf("[PowerOn] Failed to send response for device %s: %v", deviceInfo.DeviceID, err) + } +} +``` + +### 3. Device Processing Logic + +```go +// processDeviceFromPowerOn handles device identification and data updates +func (s *Server) processDeviceFromPowerOn(req *PowerOnRequest) (*models.ServiceDeviceInfo, bool, error) { + deviceMAC := req.Device.ID + deviceIP := req.DiagnosticData.DeviceLandscape.IPAddress + + // Try to find existing device by MAC address (primary identifier) + existingDevice, err := s.ds.GetDeviceByMAC(deviceMAC) + if err != nil && err != datastore.ErrDeviceNotFound { + return nil, false, fmt.Errorf("failed to lookup device: %w", err) + } + + var deviceInfo *models.ServiceDeviceInfo + isNewDevice := existingDevice == nil + + if isNewDevice { + // Create new device record from power_on data + deviceInfo = s.createDeviceFromPowerOn(req) + + // Store in datastore + if err := s.ds.SaveDeviceInfo("", deviceMAC, deviceInfo); err != nil { + return nil, false, fmt.Errorf("failed to save new device: %w", err) + } + + log.Printf("[PowerOn] New device registered: %s (IP: %s, Model: %s)", + deviceMAC, deviceIP, deviceInfo.ProductCode) + } else { + // Update existing device with power_on data + deviceInfo = existingDevice + s.updateDeviceFromPowerOn(deviceInfo, req) + + // Detect significant changes + if s.hasSignificantChanges(existingDevice, deviceInfo) { + log.Printf("[PowerOn] Device %s updated: IP %s->%s, FW %s->%s", + deviceMAC, existingDevice.IPAddress, deviceInfo.IPAddress, + existingDevice.FirmwareVersion, deviceInfo.FirmwareVersion) + } + + // Save updated device info + if err := s.ds.SaveDeviceInfo(deviceInfo.AccountID, deviceMAC, deviceInfo); err != nil { + return nil, false, fmt.Errorf("failed to update device: %w", err) + } + } + + // Update device mappings for lookup optimization + s.ds.UpdateDeviceMappings(*deviceInfo) + + return deviceInfo, isNewDevice, nil +} +``` + +### 4. Device Creation from Power-On Data + +```go +// createDeviceFromPowerOn creates a new ServiceDeviceInfo from power_on request +func (s *Server) createDeviceFromPowerOn(req *PowerOnRequest) *models.ServiceDeviceInfo { + now := time.Now() + + deviceInfo := &models.ServiceDeviceInfo{ + DeviceID: req.Device.ID, // MAC address + ProductCode: req.Device.Product.ProductCode, + DeviceSerialNumber: req.Device.SerialNumber, + ProductSerialNumber: req.Device.Product.SerialNumber, + FirmwareVersion: req.Device.FirmwareVersion, + IPAddress: req.DiagnosticData.DeviceLandscape.IPAddress, + MacAddress: req.Device.ID, // Primary MAC + DiscoveryMethod: "power_on", + LastSeen: now, + CreatedAt: now, + UpdatedAt: now, + } + + // Generate default name if not provided + if deviceInfo.Name == "" { + deviceInfo.Name = s.generateDefaultDeviceName(deviceInfo) + } + + // Add power_on specific metadata + deviceInfo.Metadata = map[string]string{ + "rssi": req.DiagnosticData.DeviceLandscape.RSSI, + "gateway_ip": req.DiagnosticData.DeviceLandscape.GatewayIP, + "connection_type": req.DiagnosticData.DeviceLandscape.ConnectionType, + "power_on_count": "1", + } + + // Store additional MAC addresses if available + if len(req.DiagnosticData.DeviceLandscape.MacAddresses) > 1 { + additionalMACs := make([]string, 0, len(req.DiagnosticData.DeviceLandscape.MacAddresses)-1) + for _, mac := range req.DiagnosticData.DeviceLandscape.MacAddresses { + if mac != req.Device.ID { + additionalMACs = append(additionalMACs, mac) + } + } + if len(additionalMACs) > 0 { + deviceInfo.Metadata["additional_macs"] = strings.Join(additionalMACs, ",") + } + } + + return deviceInfo +} +``` + +### 5. Response Generation Logic + +```go +// buildPowerOnResponse creates appropriate response based on device state +func (s *Server) buildPowerOnResponse(deviceInfo *models.ServiceDeviceInfo, isNewDevice bool, req *PowerOnRequest) *PowerOnResponse { + response := &PowerOnResponse{ + Status: "ok", + DeviceID: deviceInfo.DeviceID, + Timestamp: time.Now().Format(time.RFC3339), + } + + // Handle new device registration + if isNewDevice { + response.RegistrationRequired = deviceInfo.AccountID == "" + + // Add welcome configuration for new devices + response.ConfigurationUpdates = []ConfigurationUpdate{ + { + Type: "welcome", + Key: "device_registered", + Value: "true", + Priority: 1, + }, + } + } + + // Check if migration is needed + if s.needsMigration(deviceInfo) { + migration := s.getMigrationInstructions(deviceInfo) + response.MigrationInstructions = migration + + log.Printf("[PowerOn] Migration required for device %s: %s", + deviceInfo.DeviceID, migration.Method) + } + + // Add any pending configuration updates + pendingUpdates := s.getPendingConfigurationUpdates(deviceInfo) + response.ConfigurationUpdates = append(response.ConfigurationUpdates, pendingUpdates...) + + return response +} +``` + +### 6. Device Lookup Enhancements + +#### Enhanced DataStore Methods +```go +// GetDeviceByMAC finds a device by MAC address across all accounts +func (ds *DataStore) GetDeviceByMAC(macAddress string) (*models.ServiceDeviceInfo, error) { + normalizedMAC := normalizeMAC(macAddress) + + // Check device mappings first (for performance) + ds.idMutex.RLock() + deviceID, exists := ds.deviceMappings[normalizedMAC] + ds.idMutex.RUnlock() + + if exists { + // Try to find device by mapped ID + device, err := ds.findDeviceByID(deviceID) + if err == nil { + return device, nil + } + } + + // Fallback to full scan + devices, err := ds.ListAllDevices() + if err != nil { + return nil, err + } + + for _, device := range devices { + if normalizeMAC(device.MacAddress) == normalizedMAC || + normalizeMAC(device.DeviceID) == normalizedMAC { + return &device, nil + } + + // Check additional MAC addresses in metadata + if additionalMACs, exists := device.Metadata["additional_macs"]; exists { + for _, mac := range strings.Split(additionalMACs, ",") { + if normalizeMAC(mac) == normalizedMAC { + return &device, nil + } + } + } + } + + return nil, datastore.ErrDeviceNotFound +} +``` + +### 7. Migration Integration + +```go +// needsMigration determines if device requires configuration migration +func (s *Server) needsMigration(deviceInfo *models.ServiceDeviceInfo) bool { + if deviceInfo.AccountID == "" { + return false // Cannot migrate without account + } + + // Check if device is already migrated + if s.sm != nil { + summary, err := s.sm.GetMigrationSummary(deviceInfo.IPAddress, s.ServerURL, "", nil) + if err == nil && summary.IsMigrated { + return false + } + } + + return true +} + +// getMigrationInstructions creates migration instructions for device +func (s *Server) getMigrationInstructions(deviceInfo *models.ServiceDeviceInfo) *MigrationInstruction { + return &MigrationInstruction{ + Method: "xml", // Default to XML-based migration + TargetURL: s.ServerURL, + Options: map[string]string{ + "marge": "true", + "stats": "true", + "sw_update": "true", + }, + } +} +``` + +### 8. Error Handling and Fallbacks + +```go +// handlePowerOnError provides graceful error handling with fallbacks +func (s *Server) handlePowerOnError(w http.ResponseWriter, r *http.Request, message string, err error) { + log.Printf("[PowerOn] %s: %v", message, err) + + // Try to extract IP from request for fallback processing + if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil { + // Fallback to existing discovery mechanism + go s.PrimeDeviceWithSpotify(host) + log.Printf("[PowerOn] Falling back to legacy processing for IP %s", host) + } + + // Always return 200 OK to avoid device retry loops + w.WriteHeader(http.StatusOK) +} + +// parsePowerOnRequest safely parses the power_on request with validation +func (s *Server) parsePowerOnRequest(r *http.Request) (*PowerOnRequest, error) { + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("failed to read request body: %w", err) + } + + if len(body) == 0 { + return nil, fmt.Errorf("empty request body") + } + + var req PowerOnRequest + if err := xml.Unmarshal(body, &req); err != nil { + return nil, fmt.Errorf("failed to parse XML: %w", err) + } + + // Validate required fields + if req.Device.ID == "" { + return nil, fmt.Errorf("missing device ID") + } + + if req.DiagnosticData.DeviceLandscape.IPAddress == "" { + return nil, fmt.Errorf("missing device IP address") + } + + return &req, nil +} +``` + +### 9. Logging and Monitoring + +```go +// logPowerOnInteraction records detailed interaction logs for debugging +func (s *Server) logPowerOnInteraction(deviceInfo *models.ServiceDeviceInfo, req *PowerOnRequest, resp *PowerOnResponse, startTime time.Time) { + duration := time.Since(startTime) + + log.Printf("[PowerOn] Device: %s, IP: %s, Duration: %v, Status: %s, NewDevice: %t, Migration: %t", + deviceInfo.DeviceID, + req.DiagnosticData.DeviceLandscape.IPAddress, + duration, + resp.Status, + resp.RegistrationRequired, + resp.MigrationInstructions != nil) + + // Store interaction for debugging (if enabled) + if s.config.RecordInteractions { + interaction := models.DeviceInteraction{ + Timestamp: startTime, + DeviceID: deviceInfo.DeviceID, + Type: "power_on", + Request: req, + Response: resp, + Duration: duration, + IPAddress: req.DiagnosticData.DeviceLandscape.IPAddress, + UserAgent: r.Header.Get("User-Agent"), + } + + if err := s.ds.SaveInteraction(interaction); err != nil { + log.Printf("[PowerOn] Failed to save interaction: %v", err) + } + } +} +``` + +### 10. Configuration and Feature Flags + +```go +// PowerOnConfig controls behavior of enhanced power_on processing +type PowerOnConfig struct { + EnableEnhancedProcessing bool `json:"enable_enhanced_processing"` + AutoMigration bool `json:"auto_migration"` + RecordInteractions bool `json:"record_interactions"` + DefaultResponseTimeout time.Duration `json:"default_response_timeout"` + FallbackToLegacy bool `json:"fallback_to_legacy"` +} + +// loadPowerOnConfig loads configuration with defaults +func loadPowerOnConfig() *PowerOnConfig { + return &PowerOnConfig{ + EnableEnhancedProcessing: true, + AutoMigration: false, // Conservative default + RecordInteractions: false, + DefaultResponseTimeout: 5 * time.Second, + FallbackToLegacy: true, + } +} +``` + +## Testing Strategy + +### 1. Unit Tests +```go +func TestHandleMargePowerOnEnhanced(t *testing.T) { + tests := []struct { + name string + requestBody string + existingDevice *models.ServiceDeviceInfo + expectedStatus string + expectMigration bool + }{ + { + name: "new_device_registration", + requestBody: `...`, + existingDevice: nil, + expectedStatus: "ok", + expectMigration: false, + }, + { + name: "existing_device_update", + requestBody: `...`, + existingDevice: &models.ServiceDeviceInfo{DeviceID: "A81B6A536A98"}, + expectedStatus: "ok", + expectMigration: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test implementation + }) + } +} +``` + +### 2. Integration Tests +```go +func TestPowerOnDeviceLifecycle(t *testing.T) { + // Test complete device lifecycle through power_on events + // 1. New device power_on + // 2. Device registration + // 3. Configuration changes + // 4. Migration + // 5. Subsequent power_on events +} +``` + +### 3. Load Testing +```go +func BenchmarkPowerOnProcessing(b *testing.B) { + // Benchmark power_on processing performance + // Test concurrent device registrations + // Measure response times +} +``` + +## Deployment Strategy + +### Phase 1: Parallel Implementation +- Implement enhanced handler alongside existing handler +- Use feature flag to control which handler processes requests +- Maintain full backward compatibility + +### Phase 2: Gradual Rollout +- Enable enhanced processing for subset of devices +- Monitor performance and error rates +- Collect metrics on data completeness + +### Phase 3: Full Migration +- Default to enhanced processing for all devices +- Remove legacy fallbacks +- Optimize performance based on production data + +## Monitoring and Metrics + +### Key Metrics to Track +- Power-on event frequency per device +- New device registration rate via power_on +- Migration success rate via power_on response +- Response time distribution +- Error rates and types +- Data completeness metrics + +### Alerting Thresholds +- Power-on processing failures > 5% +- Average response time > 2 seconds +- New device registration failures > 1% +- Migration instruction delivery failures > 2% + +## Security Considerations + +### Input Validation +- XML parsing security (prevent XXE attacks) +- Device ID format validation +- IP address validation +- Request size limits + +### Authentication +- Device authentication via MAC address verification +- Request signing (if available) +- Rate limiting per device/IP + +### Data Privacy +- Sensitive data handling in diagnostic information +- Logging data retention policies +- Compliance with data protection regulations \ No newline at end of file diff --git a/docs/scmudc-events-analysis.md b/docs/scmudc-events-analysis.md new file mode 100644 index 0000000..c5fec7d --- /dev/null +++ b/docs/scmudc-events-analysis.md @@ -0,0 +1,140 @@ +# SCMUDC Events Analysis + +## Overview + +SCMUDC (Sound Control Management Usage Data Collection) events are telemetry data sent from SoundTouch devices to `events.api.bosecm.com` via `/v1/scmudc/{deviceId}` endpoints. These events track user interactions and device behaviors for analytics and monitoring. + +## Event Origins + +Analysis of recorded interactions reveals three distinct origins for device events: + +### 1. `"gabbo"` - SoundTouch App (Mobile/Desktop) +- **Source**: Remote control via SoundTouch mobile/desktop applications +- **Frequency**: Highest (primary control method) +- **Event Types**: User-initiated actions through app interface +- **Button Abstraction**: App UI elements (not physical buttons) + +**Common Events**: +- `power-pressed` β†’ Power on/off via app +- `play-pressed` β†’ Play control +- `pause-pressed` β†’ Pause control +- `skip-forward-pressed` β†’ Next track +- `stop-pressed` β†’ Stop playback + +### 2. `"console"` - Device Hardware Controls +- **Source**: Physical buttons and controls on the speaker device +- **Frequency**: Lower (secondary control method) +- **Event Types**: Direct hardware interaction +- **Physical Controls**: Actual buttons, knobs, or touch interfaces on device + +**Common Events**: +- `preset-pressed` β†’ Physical preset buttons (PRESET_1, PRESET_5, etc.) +- `power-pressed` β†’ Hardware power button + +### 3. `"device"` - Internal System Actions +- **Source**: Device's internal software systems +- **Frequency**: Automatic responses to user actions +- **Event Types**: System-generated events, content playback +- **Rich Content**: Base64-encoded XML with detailed metadata + +**Common Events**: +- `play-item` β†’ Automatic content playback responses +- `preset-assigned` β†’ System preset assignments + +## Event Data Structure + +### Standard Button Events (gabbo/console) +```json +{ + "data": { + "buttonId": "POWER|PLAY|PAUSE|PRESET_5|etc", + "origin": "gabbo|console" + }, + "type": "power-pressed|play-pressed|pause-pressed|preset-pressed|etc" +} +``` + +### Device Content Events +```json +{ + "data": { + "contentItem": "PD94bWwgdmVyc2lvbj0...", // Base64-encoded XML + "origin": "device", + "preset": "none|P1|P5|etc" + }, + "type": "play-item|preset-assigned" +} +``` + +## Content Item Structure + +Device events include Base64-encoded XML with rich content metadata: + +```xml + + Billie Eilish - bad guy (instrumental version) + https://i.scdn.co/image/ab67616d0000b273... + +``` + +**Key Fields**: +- `source`: Music service (SPOTIFY, PANDORA, etc.) +- `itemName`: Track/playlist/station name +- `sourceAccount`: User account on the service +- `location`: Service-specific content identifier +- `containerArt`: Album/playlist artwork URL +- `isPresetable`: Whether content can be saved as preset + +## Usage Patterns + +### Control Method Preferences +1. **Primary**: SoundTouch App (`gabbo`) - Most frequent interactions +2. **Secondary**: Device Hardware (`console`) - Occasional direct control +3. **Automatic**: Internal System (`device`) - Background responses + +### Event Flow +1. User triggers action via app or hardware +2. Device processes request and begins playback +3. Device sends content event with full metadata +4. System continues tracking playback state + +## Telemetry Insights + +### User Behavior Analytics +- **Interface Preference**: App vs. hardware control usage ratios +- **Feature Usage**: Most/least used controls and functions +- **Content Patterns**: Music service preferences, playlist usage + +### Device Health Monitoring +- **Interaction Frequency**: Normal vs. abnormal usage patterns +- **Error Detection**: Failed commands or unusual event sequences +- **Performance**: Response times between user action and system response + +### Service Integration Analysis +- **Music Services**: Spotify dominance, other service usage +- **Account Mapping**: User accounts across different services +- **Content Types**: Music vs. radio vs. podcast preferences + +## Data Quality Notes + +- All events include comprehensive device information (deviceID, serialNumber, softwareVersion) +- Timestamps include both UTC time and device monotonic time +- Events are batched and sent with consistent protocol versioning +- Content metadata is rich and includes artwork URLs for UI enhancement + +## Security Considerations + +- Events include user account information and listening habits +- Device serial numbers and unique identifiers are transmitted +- Content location data could reveal usage patterns +- Data should be handled according to privacy regulations + +## Technical Implementation Notes + +- Endpoint: `POST /v1/scmudc/{deviceId}` +- Protocol Version: 3.1 (current) +- Encoding: JSON with Base64-encoded XML payloads +- Authentication: Bearer token authorization +- Content-Type: `text/json; charset=utf-8`