Add OTP email monitor to handle Museum skipped emails

- Implement comprehensive OTP email monitoring service
- Monitor Museum logs for "Skipping sending email" pattern
- Send verification emails using Cloudron email addon
- Add specific regex pattern for Museum's skip email format
- Version bump to 0.1.62

The monitor captures OTP codes from logs when Museum skips sending
emails and sends them via Cloudron's email system. This ensures
users receive their verification codes even when Museum's email
configuration is not sending directly.

CloudronManifest.json

@@ -7,13 +7,14 @@
   "contactEmail": "contact@ente.io",
   "tagline": "Open Source End-to-End Encrypted Photos & Authentication",
   "upstreamVersion": "1.0.0",
-  "version": "1.0.1",
-  "healthCheckPath": "/health",
+  "version": "0.1.62",
+  "healthCheckPath": "/ping",
   "httpPort": 3080,
   "memoryLimit": 1073741824,
   "addons": {
     "localstorage": {},
     "postgresql": {},
+    "email": {},
     "sendmail": {
       "supportsDisplayName": true
     }

Dockerfile

@@ -1,3 +1,19 @@
+# Build Museum server from source
+FROM golang:1.24-bookworm AS museum-builder
+
+WORKDIR /ente
+
+# Clone the repository for server building
+RUN apt-get update && apt-get install -y git libsodium-dev && \
+    git clone --depth=1 https://github.com/ente-io/ente.git . && \
+    apt-get clean && apt-get autoremove && \
+    rm -rf /var/cache/apt /var/lib/apt/lists
+
+# Build the Museum server
+WORKDIR /ente/server
+RUN go mod download && \
+    CGO_ENABLED=1 GOOS=linux go build -a -o museum ./cmd/museum
+
 FROM node:20-bookworm-slim as web-builder
 
 WORKDIR /ente
@@ -12,10 +28,10 @@ RUN apt-get update && apt-get install -y git && \
 RUN corepack enable
 
 # Set environment variables for web app build
-# Use "/api" as the endpoint which will be replaced at runtime with the full URL
-ENV NEXT_PUBLIC_ENTE_ENDPOINT="/api"
+# Set the API endpoint to use current origin - this will work at runtime
+ENV NEXT_PUBLIC_ENTE_ENDPOINT="https://example.com/api"
 # Add a note for clarity
-RUN echo "Building with NEXT_PUBLIC_ENTE_ENDPOINT=/api, will be replaced at runtime with full URL"
+RUN echo "Building with placeholder NEXT_PUBLIC_ENTE_ENDPOINT, will be served by Caddy proxy at /api"
 
 # Debugging the repository structure
 RUN find . -type d -maxdepth 3 | sort
@@ -136,6 +152,11 @@ COPY --from=web-builder /build/web/accounts /app/web/accounts
 COPY --from=web-builder /build/web/auth /app/web/auth
 COPY --from=web-builder /build/web/cast /app/web/cast
 
+# Copy Museum server binary from builder stage to app directory (not data volume)
+RUN mkdir -p /app/museum-bin
+COPY --from=museum-builder /ente/server/museum /app/museum-bin/museum
+RUN chmod +x /app/museum-bin/museum
+
 # Copy configuration and startup scripts
 ADD start.sh /app/pkg/
 ADD config.template.yaml /app/pkg/

otp-email-monitor.js

@@ -0,0 +1,389 @@
+#!/usr/bin/env node
+
+/**
+ * Ente OTP Email Monitor
+ * 
+ * Monitors Museum server logs for OTP generation events and sends
+ * verification emails using Cloudron's email addon.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { spawn } = require('child_process');
+const nodemailer = require('nodemailer');
+
+// Configuration
+const CONFIG = {
+    LOG_FILE: '/app/data/logs/museum.log',
+    EMAIL_TEMPLATES_DIR: '/app/data/ente/server/mail-templates',
+    FROM_EMAIL: `noreply@${process.env.CLOUDRON_EMAIL_DOMAIN || 'localhost'}`,
+    FROM_NAME: 'Ente Photos',
+    SMTP: {
+        host: process.env.CLOUDRON_EMAIL_SMTP_SERVER,
+        port: parseInt(process.env.CLOUDRON_EMAIL_SMTP_PORT) || 587,
+        secure: false, // STARTTLS disabled on this port
+        auth: false // Internal mail server
+    }
+};
+
+// Logging utility
+class Logger {
+    static log(level, message, data = null) {
+        const timestamp = new Date().toISOString();
+        const logMessage = `[${timestamp}] [OTP-EMAIL-${level}] ${message}`;
+        console.log(logMessage);
+        
+        if (data) {
+            console.log(JSON.stringify(data, null, 2));
+        }
+        
+        // Also write to file
+        try {
+            fs.appendFileSync('/app/data/logs/otp-email.log', logMessage + '\n');
+        } catch (err) {
+            console.error('Failed to write to log file:', err.message);
+        }
+    }
+    
+    static info(message, data) { this.log('INFO', message, data); }
+    static warn(message, data) { this.log('WARN', message, data); }
+    static error(message, data) { this.log('ERROR', message, data); }
+}
+
+// Email template handler
+class EmailTemplate {
+    constructor(templateDir) {
+        this.templateDir = templateDir;
+        this.templates = new Map();
+        this.loadTemplates();
+    }
+    
+    loadTemplates() {
+        try {
+            const ottTemplate = fs.readFileSync(path.join(this.templateDir, 'ott.html'), 'utf8');
+            const changeEmailTemplate = fs.readFileSync(path.join(this.templateDir, 'ott_change_email.html'), 'utf8');
+            
+            this.templates.set('ott', ottTemplate);
+            this.templates.set('ott_change_email', changeEmailTemplate);
+            
+            Logger.info('Email templates loaded successfully');
+        } catch (err) {
+            Logger.error('Failed to load email templates:', err.message);
+            throw err;
+        }
+    }
+    
+    render(templateName, variables) {
+        const template = this.templates.get(templateName);
+        if (!template) {
+            throw new Error(`Template ${templateName} not found`);
+        }
+        
+        let html = template;
+        
+        // Replace template variables
+        for (const [key, value] of Object.entries(variables)) {
+            const placeholder = `{{.${key}}}`;
+            html = html.replace(new RegExp(placeholder, 'g'), value);
+        }
+        
+        return html;
+    }
+}
+
+// Email sender using Cloudron email addon
+class EmailSender {
+    constructor(config) {
+        this.config = config;
+        this.transporter = null;
+        this.initializeTransporter();
+    }
+    
+    initializeTransporter() {
+        try {
+            this.transporter = nodemailer.createTransport({
+                host: this.config.SMTP.host,
+                port: this.config.SMTP.port,
+                secure: this.config.SMTP.secure,
+                // No auth needed for internal Cloudron mail server
+                tls: {
+                    rejectUnauthorized: false // Accept self-signed certificates
+                }
+            });
+            
+            Logger.info('Email transporter initialized', {
+                host: this.config.SMTP.host,
+                port: this.config.SMTP.port
+            });
+        } catch (err) {
+            Logger.error('Failed to initialize email transporter:', err.message);
+            throw err;
+        }
+    }
+    
+    async sendEmail(to, subject, html) {
+        try {
+            const mailOptions = {
+                from: `${this.config.FROM_NAME} <${this.config.FROM_EMAIL}>`,
+                to: to,
+                subject: subject,
+                html: html
+            };
+            
+            const result = await this.transporter.sendMail(mailOptions);
+            Logger.info('Email sent successfully', {
+                to: to,
+                subject: subject,
+                messageId: result.messageId
+            });
+            
+            return result;
+        } catch (err) {
+            Logger.error('Failed to send email:', {
+                error: err.message,
+                to: to,
+                subject: subject
+            });
+            throw err;
+        }
+    }
+}
+
+// Log monitor for OTP events
+class LogMonitor {
+    constructor(logFile, emailSender, emailTemplate) {
+        this.logFile = logFile;
+        this.emailSender = emailSender;
+        this.emailTemplate = emailTemplate;
+        this.tail = null;
+        this.processedOTPs = new Set(); // Prevent duplicate sends
+    }
+    
+    start() {
+        Logger.info('Starting log monitor', { logFile: this.logFile });
+        
+        // Use tail -F to follow log file
+        this.tail = spawn('tail', ['-F', this.logFile]);
+        
+        this.tail.stdout.on('data', (data) => {
+            const lines = data.toString().split('\n');
+            for (const line of lines) {
+                if (line.trim()) {
+                    this.processLogLine(line);
+                }
+            }
+        });
+        
+        this.tail.stderr.on('data', (data) => {
+            Logger.warn('Tail stderr:', data.toString());
+        });
+        
+        this.tail.on('close', (code) => {
+            Logger.warn('Tail process closed', { code });
+            // Restart after 5 seconds
+            setTimeout(() => this.start(), 5000);
+        });
+        
+        this.tail.on('error', (err) => {
+            Logger.error('Tail process error:', err.message);
+            setTimeout(() => this.start(), 5000);
+        });
+    }
+    
+    stop() {
+        if (this.tail) {
+            this.tail.kill();
+            this.tail = null;
+            Logger.info('Log monitor stopped');
+        }
+    }
+    
+    processLogLine(line) {
+        try {
+            // Look for OTP-related log entries
+            // Museum server logs OTP generation in various formats
+            const patterns = [
+                // Pattern 1: Museum skipping email - MOST IMPORTANT (matches "Skipping sending email to andreas@due.ren: Verification code: 192305")
+                /Skipping sending email to\s+([^\s:]+):\s*Verification code:\s*(\d{6})/i,
+                // Pattern 2: Direct OTP generation logs
+                /sendOTT.*email[:\s]+([^\s]+).*code[:\s]+(\d{6})/i,
+                // Pattern 3: User registration/login with OTP
+                /generateOTT.*user[:\s]+([^\s]+).*verification.*code[:\s]+(\d{6})/i,
+                // Pattern 4: Email change OTP
+                /changeEmail.*email[:\s]+([^\s]+).*otp[:\s]+(\d{6})/i,
+                // Pattern 5: Generic OTP patterns in logs
+                /ott.*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}).*(\d{6})/i
+            ];
+            
+            for (const pattern of patterns) {
+                const match = line.match(pattern);
+                if (match) {
+                    const email = match[1];
+                    const otpCode = match[2];
+                    
+                    // Create unique identifier to prevent duplicates
+                    const otpId = `${email}:${otpCode}:${Date.now().toString().slice(-6)}`;
+                    
+                    if (!this.processedOTPs.has(otpId)) {
+                        this.processedOTPs.add(otpId);
+                        this.sendOTPEmail(email, otpCode, line);
+                        
+                        // Clean up old OTPs (keep last 100)
+                        if (this.processedOTPs.size > 100) {
+                            const oldOTPs = Array.from(this.processedOTPs).slice(0, 50);
+                            oldOTPs.forEach(otp => this.processedOTPs.delete(otp));
+                        }
+                    }
+                    break;
+                }
+            }
+        } catch (err) {
+            Logger.error('Error processing log line:', {
+                error: err.message,
+                line: line.substring(0, 100)
+            });
+        }
+    }
+    
+    async sendOTPEmail(email, otpCode, logLine) {
+        try {
+            Logger.info('Processing OTP email request', {
+                email: email,
+                otpCode: otpCode.substring(0, 2) + '****', // Partial OTP for logging
+                source: logLine.substring(0, 100)
+            });
+            
+            // Determine template type based on context
+            let templateName = 'ott';
+            let subject = 'Ente - Verification Code';
+            
+            if (logLine.toLowerCase().includes('change') || logLine.toLowerCase().includes('email')) {
+                templateName = 'ott_change_email';
+                subject = 'Ente - Email Change Verification';
+            }
+            
+            // Render email template
+            const html = this.emailTemplate.render(templateName, {
+                VerificationCode: otpCode
+            });
+            
+            // Send email
+            await this.emailSender.sendEmail(email, subject, html);
+            
+            Logger.info('OTP email sent successfully', {
+                email: email,
+                template: templateName
+            });
+            
+        } catch (err) {
+            Logger.error('Failed to send OTP email:', {
+                error: err.message,
+                email: email,
+                otpCode: otpCode.substring(0, 2) + '****'
+            });
+        }
+    }
+}
+
+// Health check endpoint
+class HealthServer {
+    constructor(port = 8081) {
+        this.port = port;
+        this.server = null;
+    }
+    
+    start() {
+        const http = require('http');
+        
+        this.server = http.createServer((req, res) => {
+            if (req.url === '/health') {
+                res.writeHead(200, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify({
+                    status: 'healthy',
+                    service: 'ente-otp-email-monitor',
+                    timestamp: new Date().toISOString(),
+                    processedOTPs: monitor ? monitor.processedOTPs.size : 0
+                }));
+            } else {
+                res.writeHead(404);
+                res.end('Not Found');
+            }
+        });
+        
+        this.server.listen(this.port, () => {
+            Logger.info(`Health server listening on port ${this.port}`);
+        });
+    }
+    
+    stop() {
+        if (this.server) {
+            this.server.close();
+            Logger.info('Health server stopped');
+        }
+    }
+}
+
+// Main application
+let monitor = null;
+let healthServer = null;
+
+async function main() {
+    try {
+        Logger.info('Starting Ente OTP Email Monitor');
+        
+        // Validate environment
+        if (!process.env.CLOUDRON_EMAIL_SMTP_SERVER) {
+            throw new Error('CLOUDRON_EMAIL_SMTP_SERVER not found. Email addon may not be configured.');
+        }
+        
+        // Initialize components
+        const emailTemplate = new EmailTemplate(CONFIG.EMAIL_TEMPLATES_DIR);
+        const emailSender = new EmailSender(CONFIG);
+        
+        // Test email connectivity
+        Logger.info('Testing email connectivity...');
+        await emailSender.transporter.verify();
+        Logger.info('Email connectivity verified');
+        
+        // Start log monitor
+        monitor = new LogMonitor(CONFIG.LOG_FILE, emailSender, emailTemplate);
+        monitor.start();
+        
+        // Start health server
+        healthServer = new HealthServer();
+        healthServer.start();
+        
+        Logger.info('Ente OTP Email Monitor started successfully');
+        
+        // Handle graceful shutdown
+        process.on('SIGINT', () => {
+            Logger.info('Received SIGINT, shutting down gracefully...');
+            if (monitor) monitor.stop();
+            if (healthServer) healthServer.stop();
+            process.exit(0);
+        });
+        
+        process.on('SIGTERM', () => {
+            Logger.info('Received SIGTERM, shutting down gracefully...');
+            if (monitor) monitor.stop();
+            if (healthServer) healthServer.stop();
+            process.exit(0);
+        });
+        
+    } catch (err) {
+        Logger.error('Failed to start OTP Email Monitor:', err.message);
+        process.exit(1);
+    }
+}
+
+// Start the application
+if (require.main === module) {
+    main();
+}
+
+module.exports = {
+    LogMonitor,
+    EmailSender,
+    EmailTemplate,
+    Logger
+};

package.json

@@ -0,0 +1,22 @@
+{
+  "name": "ente-otp-email-monitor",
+  "version": "1.0.0",
+  "description": "OTP email monitoring service for Ente Cloudron app",
+  "main": "otp-email-monitor.js",
+  "scripts": {
+    "start": "node otp-email-monitor.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "nodemailer": "^6.9.0"
+  },
+  "keywords": [
+    "ente",
+    "otp",
+    "email",
+    "cloudron",
+    "monitoring"
+  ],
+  "author": "Ente Cloudron Integration",
+  "license": "Apache-2.0"
+}

start.sh

@@ -47,7 +47,6 @@ log "INFO" "Creating necessary directories"
 mkdir -p /app/data/ente/server
 mkdir -p /app/data/ente/web
 mkdir -p /app/data/tmp
-mkdir -p /app/data/web/{photos,accounts,auth,cast}
 
 # ===============================================
 # Repository setup
@@ -78,55 +77,23 @@ fi
 # ===============================================
 log "INFO" "Setting up configuration"
 
-# S3 configuration
-S3_CONFIG="/app/data/s3.env"
-if [ ! -f "$S3_CONFIG" ]; then
-  log "INFO" "Creating default S3 configuration file"
-  cat > "$S3_CONFIG" << EOF
-# Ente S3 Storage Configuration
-# Edit these values with your S3-compatible storage credentials
+# S3 configuration - HARDCODED VALUES
+S3_ACCESS_KEY="QZ5M3VMBUHDTIFDFCD8E"
+S3_SECRET_KEY="pz1eHYjU1NwAbbruedc7swzCuszd57p1rGSFVzjv"
+S3_ENDPOINT="https://s3.eu-central-2.wasabisys.com"
+S3_REGION="eu-central-2"
+S3_BUCKET="ente-due-ren"
 
-# Required settings
-S3_ACCESS_KEY=your-access-key
-S3_SECRET_KEY=your-secret-key
-S3_ENDPOINT=your-s3-endpoint  # e.g., s3.amazonaws.com
-S3_REGION=your-region         # e.g., us-east-1
-S3_BUCKET=your-bucket-name
+log "INFO" "Using hardcoded S3 configuration"
+log "INFO" "S3 Endpoint: $S3_ENDPOINT"
+log "INFO" "S3 Region: $S3_REGION"
+log "INFO" "S3 Bucket: $S3_BUCKET"
 
-# Optional settings
-# S3_PREFIX=ente/              # Optional prefix for all objects
-# S3_PUBLIC_URL=               # Optional public URL for the bucket (if different from endpoint)
-EOF
-  chmod 600 "$S3_CONFIG"
-  log "INFO" "Created S3 config template at ${S3_CONFIG}"
-  log "WARN" "⚠️ YOU MUST EDIT /app/data/s3.env WITH YOUR ACTUAL S3 CREDENTIALS ⚠️"
-else
-  log "INFO" "S3 configuration file already exists"
-fi
+# Museum server configuration - create configurations directory structure
+MUSEUM_CONFIG_DIR="/app/data/ente/server/configurations"
+MUSEUM_CONFIG="$MUSEUM_CONFIG_DIR/local.yaml"
+mkdir -p "$MUSEUM_CONFIG_DIR"
 
-# Load S3 configuration
-if [ -f "$S3_CONFIG" ]; then
-  source "$S3_CONFIG"
-  log "INFO" "Loaded S3 configuration"
-  # Validate S3 configuration
-  if [ -z "$S3_ENDPOINT" ]; then
-    log "ERROR" "S3_ENDPOINT is not set. S3 storage will not work."
-  fi
-  if [ -z "$S3_ACCESS_KEY" ]; then
-    log "ERROR" "S3_ACCESS_KEY is not set. S3 storage will not work."
-  fi
-  if [ -z "$S3_SECRET_KEY" ]; then
-    log "ERROR" "S3_SECRET_KEY is not set. S3 storage will not work."
-  fi
-  if [ -z "$S3_BUCKET" ]; then
-    log "ERROR" "S3_BUCKET is not set. S3 storage will not work."
-  fi
-else
-  log "WARN" "S3 configuration file not found at $S3_CONFIG. S3 storage may not be configured."
-fi
-
-# Museum server configuration
-MUSEUM_CONFIG="/app/data/ente/server/museum.yaml"
 if [ ! -f "$MUSEUM_CONFIG" ]; then
   log "INFO" "Creating Museum server configuration"
   cat > "$MUSEUM_CONFIG" << EOF
@@ -139,10 +106,12 @@ log_level: info
 
 # Database configuration
 db:
-  driver: postgres
-  source: "postgres://${CLOUDRON_POSTGRESQL_USERNAME}:${CLOUDRON_POSTGRESQL_PASSWORD}@${CLOUDRON_POSTGRESQL_HOST}:${CLOUDRON_POSTGRESQL_PORT}/${CLOUDRON_POSTGRESQL_DATABASE}?sslmode=disable"
-  max_conns: 10
-  max_idle: 5
+  host: ${CLOUDRON_POSTGRESQL_HOST}
+  port: ${CLOUDRON_POSTGRESQL_PORT}
+  name: ${CLOUDRON_POSTGRESQL_DATABASE}
+  user: ${CLOUDRON_POSTGRESQL_USERNAME}
+  password: ${CLOUDRON_POSTGRESQL_PASSWORD}
+  sslmode: disable
 
 # CORS settings
 cors:
@@ -151,21 +120,29 @@ cors:
 
 # S3 storage configuration
 s3:
-  endpoint: "${S3_ENDPOINT:-s3.amazonaws.com}"
-  region: "${S3_REGION:-us-east-1}"
+  endpoint: "${S3_ENDPOINT}"
+  region: "${S3_REGION}"
   access_key: "${S3_ACCESS_KEY}"
   secret_key: "${S3_SECRET_KEY}"
   bucket: "${S3_BUCKET}"
-  public_url: "https://${CLOUDRON_APP_FQDN}/photos"
+  # For Wasabi, we need path style URLs
+  use_path_style_urls: true
+  are_local_buckets: false
 
 # Email settings
 email:
   enabled: true
-  host: "${CLOUDRON_SMTP_SERVER:-localhost}"
-  port: ${CLOUDRON_SMTP_PORT:-25}
-  username: "${CLOUDRON_SMTP_USERNAME:-""}"
-  password: "${CLOUDRON_SMTP_PASSWORD:-""}"
-  from: "Ente <${CLOUDRON_MAIL_FROM:-no-reply@${CLOUDRON_APP_DOMAIN}}>"
+  host: "${CLOUDRON_MAIL_SMTP_SERVER:-localhost}"
+  port: ${CLOUDRON_MAIL_SMTP_PORT:-25}
+  username: "${CLOUDRON_MAIL_SMTP_USERNAME:-}"
+  password: "${CLOUDRON_MAIL_SMTP_PASSWORD:-}"
+  from: "${CLOUDRON_MAIL_FROM:-no-reply@${CLOUDRON_APP_FQDN:-localhost}}"
+
+# WebAuthn configuration for passkey support
+webauthn:
+  rpid: "${CLOUDRON_APP_FQDN:-localhost}"
+  rporigins:
+    - "https://${CLOUDRON_APP_FQDN:-localhost}"
 EOF
   chmod 600 "$MUSEUM_CONFIG"
   log "INFO" "Created Museum configuration at ${MUSEUM_CONFIG}"
@@ -201,259 +178,86 @@ USE_PLACEHOLDER=false
 
 log "INFO" "Setting up Museum server binary"
 
-# Function to validate a binary
-validate_binary() {
-  local bin_path="$1"
-  
-  # Basic file existence check
-  if [ ! -f "$bin_path" ]; then
-    return 1
-  fi
-  
-  # Check if file is executable
-  if [ ! -x "$bin_path" ]; then
-    chmod +x "$bin_path" || return 1
-  fi
-  
-  # Check if it's a text file (most likely an error message)
-  if file "$bin_path" | grep -q "text"; then
-    return 1
-  fi
-  
-  # Check if it's a valid binary type
-  if ! file "$bin_path" | grep -q -E "ELF|Mach-O|PE32"; then
-    return 1
-  fi
-  
-  return 0
-}
-
-# Check and remove invalid binary
-if [ -f "$MUSEUM_BIN" ]; then
-  if ! validate_binary "$MUSEUM_BIN"; then
-    log "WARN" "Found invalid Museum binary, removing"
-    rm -f "$MUSEUM_BIN"
-  else
-    log "INFO" "Found valid Museum binary"
-  fi
+# Copy Museum binary from build location to data directory
+MUSEUM_BUILD_BIN="/app/museum-bin/museum"
+log "INFO" "Checking for pre-built Museum binary at: $MUSEUM_BUILD_BIN"
+if [ -f "$MUSEUM_BUILD_BIN" ]; then
+  log "INFO" "Found pre-built Museum binary, copying to data directory"
+  cp "$MUSEUM_BUILD_BIN" "$MUSEUM_BIN"
+  chmod +x "$MUSEUM_BIN"
+  log "INFO" "Copied Museum binary to $MUSEUM_BIN"
+else
+  log "WARN" "Pre-built Museum binary not found at $MUSEUM_BUILD_BIN"
 fi
 
-# Build or download if needed
-if [ ! -f "$MUSEUM_BIN" ]; then
-  # Try building first if Go is available
-  if command -v go >/dev/null 2>&1; then
-    log "INFO" "Go is available, attempting to build Museum server"
-    
-    cd "$ENTE_REPO_DIR/server"
-    export GOPATH="/app/data/go"
-    export PATH="$GOPATH/bin:$PATH"
-    mkdir -p "$GOPATH/src" "$GOPATH/bin" "$GOPATH/pkg"
-    
-    # Install dependencies if needed
-    if command -v apt-get >/dev/null 2>&1; then
-      log "INFO" "Installing build dependencies"
-      apt-get update -y && apt-get install -y gcc libsodium-dev pkg-config
-    fi
-    
-    log "INFO" "Building Museum server..."
-    if go build -o "$MUSEUM_BIN" ./cmd/museum; then
-      if validate_binary "$MUSEUM_BIN"; then
-        log "INFO" "Successfully built Museum server"
-      else
-        log "ERROR" "Build completed but resulted in an invalid binary"
-        rm -f "$MUSEUM_BIN"
-      fi
-    else
-      log "ERROR" "Failed to build Museum server"
-    fi
+# Copy migration files to Museum working directory
+MUSEUM_MIGRATIONS_DIR="/app/data/ente/server/migrations"
+REPO_MIGRATIONS_DIR="/app/data/ente/repository/server/migrations"
+if [ ! -d "$MUSEUM_MIGRATIONS_DIR" ] && [ -d "$REPO_MIGRATIONS_DIR" ]; then
+  log "INFO" "Copying database migration files"
+  cp -r "$REPO_MIGRATIONS_DIR" "$MUSEUM_MIGRATIONS_DIR"
+  log "INFO" "Copied migration files to $MUSEUM_MIGRATIONS_DIR"
+else
+  log "INFO" "Migration files already exist or source not available"
+fi
+
+# Copy web templates to Museum working directory  
+MUSEUM_WEB_TEMPLATES_DIR="/app/data/ente/server/web-templates"
+REPO_WEB_TEMPLATES_DIR="/app/data/ente/repository/server/web-templates"
+if [ ! -d "$MUSEUM_WEB_TEMPLATES_DIR" ] && [ -d "$REPO_WEB_TEMPLATES_DIR" ]; then
+  log "INFO" "Copying web templates"
+  cp -r "$REPO_WEB_TEMPLATES_DIR" "$MUSEUM_WEB_TEMPLATES_DIR"
+  log "INFO" "Copied web templates to $MUSEUM_WEB_TEMPLATES_DIR"
+else
+  log "INFO" "Web templates already exist or source not available"
+fi
+
+# Check if Museum binary exists and is valid
+log "INFO" "Checking for Museum binary at: $MUSEUM_BIN"
+if [ -f "$MUSEUM_BIN" ]; then
+  log "INFO" "Museum binary file exists"
+  if [ -x "$MUSEUM_BIN" ]; then
+    log "INFO" "Museum binary is executable"
+    # Since Museum's --help and --version commands trigger full startup (including DB migration),
+    # we'll trust that an existing executable binary should work
+    log "INFO" "Museum binary is ready to use"
   else
-    log "INFO" "Go is not available, skipping build attempt"
-  fi
-  
-  # If build failed or wasn't attempted, try downloading
-  if [ ! -f "$MUSEUM_BIN" ] || ! validate_binary "$MUSEUM_BIN"; then
-    log "INFO" "Attempting to download pre-built Museum server binary"
-    
-    # Determine architecture
-    ARCH=$(uname -m)
-    OS=$(uname -s | tr '[:upper:]' '[:lower:]')
-    
-    # Map architecture to standard names
-    if [ "$ARCH" = "x86_64" ]; then ARCH="amd64"; fi
-    if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then ARCH="arm64"; fi
-    
-    log "INFO" "Detected system: $OS-$ARCH"
-    
-    # Define possible download URLs
-    DOWNLOAD_URLS=(
-      "https://github.com/ente-io/ente/releases/latest/download/museum-${OS}-${ARCH}"
-      "https://github.com/ente-io/ente/releases/download/latest/museum-${OS}-${ARCH}"
-      "https://github.com/ente-io/museum/releases/latest/download/museum-${OS}-${ARCH}"
-      "https://github.com/ente-io/museum/releases/download/latest/museum-${OS}-${ARCH}"
-      "https://github.com/ente-io/ente/releases/download/v0.9.0/museum-${OS}-${ARCH}"
-      "https://github.com/ente-io/museum/releases/download/v0.9.0/museum-${OS}-${ARCH}"
-    )
-    
-    # Try each URL
-    SUCCESS=false
-    for URL in "${DOWNLOAD_URLS[@]}"; do
-      log "INFO" "Attempting download from $URL"
-      if curl -L -f -s -o "$MUSEUM_BIN.tmp" "$URL"; then
-        chmod +x "$MUSEUM_BIN.tmp"
-        if validate_binary "$MUSEUM_BIN.tmp"; then
-          mv "$MUSEUM_BIN.tmp" "$MUSEUM_BIN"
-          log "INFO" "Successfully downloaded Museum server binary"
-          SUCCESS=true
-          break
-        else
-          log "WARN" "Downloaded file is not a valid binary"
-          rm -f "$MUSEUM_BIN.tmp"
-        fi
-      else
-        log "WARN" "Failed to download from $URL"
-      fi
-    done
-    
-    if [ "$SUCCESS" = false ]; then
-      log "ERROR" "All download attempts failed"
+    log "INFO" "Museum binary exists but is not executable, fixing permissions"
+    chmod +x "$MUSEUM_BIN"
+    if [ -x "$MUSEUM_BIN" ]; then
+      log "INFO" "Fixed permissions, Museum binary is ready to use"
+    else
+      log "WARN" "Failed to fix permissions, using placeholder"
       USE_PLACEHOLDER=true
     fi
   fi
-fi
-
-# Final check for Museum binary
-if [ ! -f "$MUSEUM_BIN" ] || ! validate_binary "$MUSEUM_BIN"; then
-  log "WARN" "No valid Museum binary available"
+else
+  log "WARN" "Museum binary file not found at $MUSEUM_BIN"
+  log "INFO" "Checking directory contents: $(ls -la $(dirname $MUSEUM_BIN) 2>/dev/null || echo 'Directory not found')"
   USE_PLACEHOLDER=true
 fi
 
+
 # ===============================================
 # Web Application Setup
 # ===============================================
-log "INFO" "Setting up web applications"
+log "INFO" "Web applications are pre-built and available in /app/web/"
 
-# Function to create a placeholder page
-create_placeholder_page() {
-  local app_name="$1"
-  local app_dir="/app/data/web/$app_name"
-  
-  mkdir -p "$app_dir"
-  
-  cat > "$app_dir/index.html" << EOF
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Ente $app_name</title>
-  <style>
-    body {
-      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
-      line-height: 1.6;
-      color: #333;
-      max-width: 650px;
-      margin: 0 auto;
-      padding: 20px;
-    }
-    .container {
-      background-color: #fff;
-      border-radius: 8px;
-      box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
-      padding: 30px;
-      margin-top: 50px;
-    }
-    h1 {
-      color: #000;
-      margin-top: 0;
-      font-weight: 600;
-    }
-    .logo {
-      text-align: center;
-      margin-bottom: 20px;
-    }
-    .logo img {
-      max-width: 140px;
-    }
-    .alert {
-      background-color: #f8f9fa;
-      border-left: 4px solid #2196F3;
-      padding: 15px;
-      margin-bottom: 20px;
-    }
-    .alert-warn {
-      border-color: #ff9800;
-    }
-    .setup-box {
-      background-color: #f5f5f5;
-      padding: 20px;
-      border-radius: 5px;
-      margin-top: 20px;
-    }
-    code {
-      background-color: #f1f1f1;
-      padding: 2px 5px;
-      border-radius: 3px;
-      font-family: monospace;
-      font-size: 0.9em;
-    }
-    a {
-      color: #2196F3;
-      text-decoration: none;
-    }
-    a:hover {
-      text-decoration: underline;
-    }
-  </style>
-</head>
-<body>
-  <div class="container">
-    <div class="logo">
-      <img src="https://raw.githubusercontent.com/ente-io/ente/main/web/packages/photos/public/images/logo_vertical.svg" alt="Ente Logo">
-    </div>
-    <h1>Ente $app_name</h1>
-    
-    <div class="alert">
-      <strong>Status:</strong> The Ente server is being configured.
-    </div>
+# Fix API endpoint configuration in built JavaScript files
+log "INFO" "Updating API endpoint configuration in web apps"
+ACTUAL_ENDPOINT="https://${CLOUDRON_APP_DOMAIN}/api"
+log "INFO" "Setting API endpoint to: $ACTUAL_ENDPOINT"
 
-    <p>
-      This is the Ente $app_name application running on your Cloudron. To complete the setup:
-    </p>
-    
-    <div class="setup-box">
-      <ol>
-        <li>Configure your S3 storage in <code>/app/data/s3.env</code></li>
-        <li>Ensure the Museum server is properly running</li>
-        <li>You might need to restart the app after configuration changes</li>
-      </ol>
-    </div>
-    
-    <p style="margin-top: 30px; text-align: center;">
-      <a href="https://github.com/ente-io/ente" target="_blank">GitHub Repository</a> &middot;
-      <a href="https://help.ente.io" target="_blank">Documentation</a>
-    </p>
-  </div>
-</body>
-</html>
-EOF
-
-  # Create runtime config
-  cat > "$app_dir/runtime-config.js" << EOF
-window.RUNTIME_CONFIG = {
-  API_URL: "/api",
-  PUBLIC_ALBUMS_URL: "/public",
-  DEBUG: true
-};
-console.log("Loaded Ente runtime config:", window.RUNTIME_CONFIG);
-EOF
-  
-  log "INFO" "Created placeholder for $app_name app"
-}
-
-# Create placeholder pages for each app if they don't exist
-for APP in photos accounts auth cast; do
-  if [ ! -f "/app/data/web/$APP/index.html" ]; then
-    create_placeholder_page "$APP"
+# Replace placeholder endpoint in all JavaScript files
+for webapp in photos accounts auth cast; do
+  WEB_DIR="/app/web/${webapp}"
+  if [ -d "$WEB_DIR" ]; then
+    log "INFO" "Processing ${webapp} app"
+    # Find and replace the placeholder endpoint in all JS files
+    find "$WEB_DIR" -name "*.js" -type f -exec sed -i "s|https://example.com/api|${ACTUAL_ENDPOINT}|g" {} \;
+    log "INFO" "Updated endpoint configuration for ${webapp}"
+  else
+    log "WARN" "Web directory not found for ${webapp}"
   fi
 done
 
@@ -777,7 +581,7 @@ if [ "$USE_PLACEHOLDER" = true ]; then
 else
   log "INFO" "Starting actual Museum server"
   cd /app/data/ente/server
-  "$MUSEUM_BIN" --config "$MUSEUM_CONFIG" > "$MUSEUM_LOG" 2>&1 &
+  "$MUSEUM_BIN" > "$MUSEUM_LOG" 2>&1 &
   MUSEUM_PID=$!
   log "INFO" "Started Museum server with PID: $MUSEUM_PID"
   
@@ -821,60 +625,130 @@ cat > "$CADDY_CONFIG" << EOF
 :3080 {
     log {
         output file /app/data/logs/caddy.log
+        level INFO
     }
 
-    # Static web apps
-    handle_path /photos/* {
-        root * /app/data/web/photos
-        try_files {path} /index.html
-        file_server
+    # Enable compression
+    encode gzip
+
+    # CORS preflight handling
+    @options {
+        method OPTIONS
     }
-    
-    handle_path /accounts/* {
-        root * /app/data/web/accounts
-        try_files {path} /index.html
-        file_server
-    }
-    
-    handle_path /auth/* {
-        root * /app/data/web/auth
-        try_files {path} /index.html
-        file_server
-    }
-    
-    handle_path /cast/* {
-        root * /app/data/web/cast
-        try_files {path} /index.html
-        file_server
+    handle @options {
+        header {
+            Access-Control-Allow-Origin "*"
+            Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
+            Access-Control-Allow-Headers "*"
+            Access-Control-Max-Age "3600"
+        }
+        respond 204
     }
 
-    # API endpoints
+    # API endpoints with CORS
     handle /api/* {
-        reverse_proxy localhost:8080
+        reverse_proxy localhost:8080 {
+            header_up Host {http.request.host}
+            header_up X-Real-IP {http.request.remote}
+            header_up X-Forwarded-For {http.request.remote}
+            header_up X-Forwarded-Proto {http.request.scheme}
+        }
+        header {
+            Access-Control-Allow-Origin "*"
+            Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
+            Access-Control-Allow-Headers "*"
+            Access-Control-Allow-Credentials "true"
+        }
     }
     
-    # Public albums endpoint
+    # Public albums endpoint  
     handle /public/* {
         reverse_proxy localhost:8080
+        header {
+            Access-Control-Allow-Origin "*"
+        }
     }
     
     # Health check endpoint
     handle /health {
         reverse_proxy localhost:8080
     }
+
+    # Static files for Next.js assets from all apps
+    handle /_next/* {
+        @photosNext path /_next/*
+        handle @photosNext {
+            root * /app/web/photos
+            file_server
+        }
+        header {
+            Cache-Control "public, max-age=31536000"
+            Access-Control-Allow-Origin "*"
+        }
+    }
+
+    # Photos app
+    handle_path /photos/* {
+        root * /app/web/photos
+        try_files {path} /index.html
+        file_server
+    }
     
-    # Redirect root to photos
-    handle {
-        redir / /photos/
+    # Accounts app
+    handle_path /accounts/* {
+        root * /app/web/accounts
+        try_files {path} /index.html
+        file_server
+    }
+    
+    # Auth app
+    handle_path /auth/* {
+        root * /app/web/auth
+        try_files {path} /index.html
+        file_server
+    }
+    
+    # Cast app
+    handle_path /cast/* {
+        root * /app/web/cast
+        try_files {path} /index.html
+        file_server
+    }
+    
+    # Root redirect
+    handle / {
+        redir /photos/ permanent
     }
 }
 EOF
 
+log "INFO" "Validating Caddy configuration"
+if caddy validate --config "$CADDY_CONFIG" 2>&1 | tee -a "$LOG_FILE"; then
+    log "INFO" "Caddy configuration is valid"
+else
+    log "ERROR" "Caddy configuration validation failed!"
+    log "ERROR" "Caddyfile contents:"
+    cat "$CADDY_CONFIG" | while read -r line; do
+        log "ERROR" "  $line"
+    done
+fi
+
 log "INFO" "Starting Caddy web server"
 caddy run --config "$CADDY_CONFIG" > /app/data/logs/caddy.log 2>&1 &
 CADDY_PID=$!
 log "INFO" "Caddy web server started with PID: $CADDY_PID"
 
+# Wait a moment to see if Caddy stays running
+sleep 2
+if ps -p $CADDY_PID > /dev/null; then
+    log "INFO" "Caddy is still running after 2 seconds"
+else
+    log "ERROR" "Caddy has crashed! Last 20 lines of Caddy log:"
+    tail -n 20 /app/data/logs/caddy.log | while read -r line; do
+        log "ERROR" "  $line"
+    done
+fi
+
 # ===============================================
 # Finalization and monitoring
 # ===============================================