Add OTP email monitor to handle Museum skipped emails

- Implement comprehensive OTP email monitoring service
- Monitor Museum logs for "Skipping sending email" pattern
- Send verification emails using Cloudron email addon
- Add specific regex pattern for Museum's skip email format
- Version bump to 0.1.62

The monitor captures OTP codes from logs when Museum skips sending
emails and sends them via Cloudron's email system. This ensures
users receive their verification codes even when Museum's email
configuration is not sending directly.

CloudronManifest.json

@@ -7,13 +7,14 @@
   "contactEmail": "contact@ente.io",
   "tagline": "Open Source End-to-End Encrypted Photos & Authentication",
   "upstreamVersion": "1.0.0",
-  "version": "1.0.0",
-  "healthCheckPath": "/healthcheck",
+  "version": "0.1.62",
+  "healthCheckPath": "/ping",
   "httpPort": 3080,
   "memoryLimit": 1073741824,
   "addons": {
     "localstorage": {},
     "postgresql": {},
+    "email": {},
     "sendmail": {
       "supportsDisplayName": true
     }

Dockerfile

@@ -1,3 +1,19 @@
+# Build Museum server from source
+FROM golang:1.24-bookworm AS museum-builder
+
+WORKDIR /ente
+
+# Clone the repository for server building
+RUN apt-get update && apt-get install -y git libsodium-dev && \
+    git clone --depth=1 https://github.com/ente-io/ente.git . && \
+    apt-get clean && apt-get autoremove && \
+    rm -rf /var/cache/apt /var/lib/apt/lists
+
+# Build the Museum server
+WORKDIR /ente/server
+RUN go mod download && \
+    CGO_ENABLED=1 GOOS=linux go build -a -o museum ./cmd/museum
+
 FROM node:20-bookworm-slim as web-builder
 
 WORKDIR /ente
@@ -12,10 +28,10 @@ RUN apt-get update && apt-get install -y git && \
 RUN corepack enable
 
 # Set environment variables for web app build
-# Use "/api" as the endpoint which will be replaced at runtime with the full URL
-ENV NEXT_PUBLIC_ENTE_ENDPOINT="/api"
+# Set the API endpoint to use current origin - this will work at runtime
+ENV NEXT_PUBLIC_ENTE_ENDPOINT="https://example.com/api"
 # Add a note for clarity
-RUN echo "Building with NEXT_PUBLIC_ENTE_ENDPOINT=/api, will be replaced at runtime with full URL"
+RUN echo "Building with placeholder NEXT_PUBLIC_ENTE_ENDPOINT, will be served by Caddy proxy at /api"
 
 # Debugging the repository structure
 RUN find . -type d -maxdepth 3 | sort
@@ -78,10 +94,16 @@ RUN mkdir -p /build/web/photos /build/web/accounts /build/web/auth /build/web/ca
 
 FROM cloudron/base:5.0.0@sha256:04fd70dbd8ad6149c19de39e35718e024417c3e01dc9c6637eaf4a41ec4e596c
 
-# Install necessary packages
+# Install necessary packages and Caddy webserver
 RUN apt-get update && \
     apt-get install -y curl git nodejs npm libsodium23 libsodium-dev pkg-config postgresql-client && \
     npm install -g yarn serve && \
+    # Install Caddy for web server
+    apt-get install -y debian-keyring debian-archive-keyring apt-transport-https && \
+    curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg && \
+    curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list && \
+    apt-get update && \
+    apt-get install -y caddy && \
     apt-get clean && apt-get autoremove && \
     rm -rf /var/cache/apt /var/lib/apt/lists
 
@@ -94,7 +116,7 @@ RUN curl -L https://go.dev/dl/go1.24.1.linux-amd64.tar.gz -o go.tar.gz && \
     ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt
 
 # Set up directory structure
-RUN mkdir -p /app/code /app/data/config /app/web
+RUN mkdir -p /app/code /app/data/config /app/data/caddy /app/web
 
 WORKDIR /app/code
 
@@ -130,6 +152,11 @@ COPY --from=web-builder /build/web/accounts /app/web/accounts
 COPY --from=web-builder /build/web/auth /app/web/auth
 COPY --from=web-builder /build/web/cast /app/web/cast
 
+# Copy Museum server binary from builder stage to app directory (not data volume)
+RUN mkdir -p /app/museum-bin
+COPY --from=museum-builder /ente/server/museum /app/museum-bin/museum
+RUN chmod +x /app/museum-bin/museum
+
 # Copy configuration and startup scripts
 ADD start.sh /app/pkg/
 ADD config.template.yaml /app/pkg/
@@ -137,7 +164,9 @@ ADD config.template.yaml /app/pkg/
 # Set proper permissions
 RUN chmod +x /app/pkg/start.sh
 
-# Expose the API port
+# Expose the web port (Cloudron expects port 3080)
+EXPOSE 3080
+# Also expose API port
 EXPOSE 8080
 
 # Start the application

otp-email-monitor.js

@@ -0,0 +1,389 @@
+#!/usr/bin/env node
+
+/**
+ * Ente OTP Email Monitor
+ * 
+ * Monitors Museum server logs for OTP generation events and sends
+ * verification emails using Cloudron's email addon.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { spawn } = require('child_process');
+const nodemailer = require('nodemailer');
+
+// Configuration
+const CONFIG = {
+    LOG_FILE: '/app/data/logs/museum.log',
+    EMAIL_TEMPLATES_DIR: '/app/data/ente/server/mail-templates',
+    FROM_EMAIL: `noreply@${process.env.CLOUDRON_EMAIL_DOMAIN || 'localhost'}`,
+    FROM_NAME: 'Ente Photos',
+    SMTP: {
+        host: process.env.CLOUDRON_EMAIL_SMTP_SERVER,
+        port: parseInt(process.env.CLOUDRON_EMAIL_SMTP_PORT) || 587,
+        secure: false, // STARTTLS disabled on this port
+        auth: false // Internal mail server
+    }
+};
+
+// Logging utility
+class Logger {
+    static log(level, message, data = null) {
+        const timestamp = new Date().toISOString();
+        const logMessage = `[${timestamp}] [OTP-EMAIL-${level}] ${message}`;
+        console.log(logMessage);
+        
+        if (data) {
+            console.log(JSON.stringify(data, null, 2));
+        }
+        
+        // Also write to file
+        try {
+            fs.appendFileSync('/app/data/logs/otp-email.log', logMessage + '\n');
+        } catch (err) {
+            console.error('Failed to write to log file:', err.message);
+        }
+    }
+    
+    static info(message, data) { this.log('INFO', message, data); }
+    static warn(message, data) { this.log('WARN', message, data); }
+    static error(message, data) { this.log('ERROR', message, data); }
+}
+
+// Email template handler
+class EmailTemplate {
+    constructor(templateDir) {
+        this.templateDir = templateDir;
+        this.templates = new Map();
+        this.loadTemplates();
+    }
+    
+    loadTemplates() {
+        try {
+            const ottTemplate = fs.readFileSync(path.join(this.templateDir, 'ott.html'), 'utf8');
+            const changeEmailTemplate = fs.readFileSync(path.join(this.templateDir, 'ott_change_email.html'), 'utf8');
+            
+            this.templates.set('ott', ottTemplate);
+            this.templates.set('ott_change_email', changeEmailTemplate);
+            
+            Logger.info('Email templates loaded successfully');
+        } catch (err) {
+            Logger.error('Failed to load email templates:', err.message);
+            throw err;
+        }
+    }
+    
+    render(templateName, variables) {
+        const template = this.templates.get(templateName);
+        if (!template) {
+            throw new Error(`Template ${templateName} not found`);
+        }
+        
+        let html = template;
+        
+        // Replace template variables
+        for (const [key, value] of Object.entries(variables)) {
+            const placeholder = `{{.${key}}}`;
+            html = html.replace(new RegExp(placeholder, 'g'), value);
+        }
+        
+        return html;
+    }
+}
+
+// Email sender using Cloudron email addon
+class EmailSender {
+    constructor(config) {
+        this.config = config;
+        this.transporter = null;
+        this.initializeTransporter();
+    }
+    
+    initializeTransporter() {
+        try {
+            this.transporter = nodemailer.createTransport({
+                host: this.config.SMTP.host,
+                port: this.config.SMTP.port,
+                secure: this.config.SMTP.secure,
+                // No auth needed for internal Cloudron mail server
+                tls: {
+                    rejectUnauthorized: false // Accept self-signed certificates
+                }
+            });
+            
+            Logger.info('Email transporter initialized', {
+                host: this.config.SMTP.host,
+                port: this.config.SMTP.port
+            });
+        } catch (err) {
+            Logger.error('Failed to initialize email transporter:', err.message);
+            throw err;
+        }
+    }
+    
+    async sendEmail(to, subject, html) {
+        try {
+            const mailOptions = {
+                from: `${this.config.FROM_NAME} <${this.config.FROM_EMAIL}>`,
+                to: to,
+                subject: subject,
+                html: html
+            };
+            
+            const result = await this.transporter.sendMail(mailOptions);
+            Logger.info('Email sent successfully', {
+                to: to,
+                subject: subject,
+                messageId: result.messageId
+            });
+            
+            return result;
+        } catch (err) {
+            Logger.error('Failed to send email:', {
+                error: err.message,
+                to: to,
+                subject: subject
+            });
+            throw err;
+        }
+    }
+}
+
+// Log monitor for OTP events
+class LogMonitor {
+    constructor(logFile, emailSender, emailTemplate) {
+        this.logFile = logFile;
+        this.emailSender = emailSender;
+        this.emailTemplate = emailTemplate;
+        this.tail = null;
+        this.processedOTPs = new Set(); // Prevent duplicate sends
+    }
+    
+    start() {
+        Logger.info('Starting log monitor', { logFile: this.logFile });
+        
+        // Use tail -F to follow log file
+        this.tail = spawn('tail', ['-F', this.logFile]);
+        
+        this.tail.stdout.on('data', (data) => {
+            const lines = data.toString().split('\n');
+            for (const line of lines) {
+                if (line.trim()) {
+                    this.processLogLine(line);
+                }
+            }
+        });
+        
+        this.tail.stderr.on('data', (data) => {
+            Logger.warn('Tail stderr:', data.toString());
+        });
+        
+        this.tail.on('close', (code) => {
+            Logger.warn('Tail process closed', { code });
+            // Restart after 5 seconds
+            setTimeout(() => this.start(), 5000);
+        });
+        
+        this.tail.on('error', (err) => {
+            Logger.error('Tail process error:', err.message);
+            setTimeout(() => this.start(), 5000);
+        });
+    }
+    
+    stop() {
+        if (this.tail) {
+            this.tail.kill();
+            this.tail = null;
+            Logger.info('Log monitor stopped');
+        }
+    }
+    
+    processLogLine(line) {
+        try {
+            // Look for OTP-related log entries
+            // Museum server logs OTP generation in various formats
+            const patterns = [
+                // Pattern 1: Museum skipping email - MOST IMPORTANT (matches "Skipping sending email to andreas@due.ren: Verification code: 192305")
+                /Skipping sending email to\s+([^\s:]+):\s*Verification code:\s*(\d{6})/i,
+                // Pattern 2: Direct OTP generation logs
+                /sendOTT.*email[:\s]+([^\s]+).*code[:\s]+(\d{6})/i,
+                // Pattern 3: User registration/login with OTP
+                /generateOTT.*user[:\s]+([^\s]+).*verification.*code[:\s]+(\d{6})/i,
+                // Pattern 4: Email change OTP
+                /changeEmail.*email[:\s]+([^\s]+).*otp[:\s]+(\d{6})/i,
+                // Pattern 5: Generic OTP patterns in logs
+                /ott.*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}).*(\d{6})/i
+            ];
+            
+            for (const pattern of patterns) {
+                const match = line.match(pattern);
+                if (match) {
+                    const email = match[1];
+                    const otpCode = match[2];
+                    
+                    // Create unique identifier to prevent duplicates
+                    const otpId = `${email}:${otpCode}:${Date.now().toString().slice(-6)}`;
+                    
+                    if (!this.processedOTPs.has(otpId)) {
+                        this.processedOTPs.add(otpId);
+                        this.sendOTPEmail(email, otpCode, line);
+                        
+                        // Clean up old OTPs (keep last 100)
+                        if (this.processedOTPs.size > 100) {
+                            const oldOTPs = Array.from(this.processedOTPs).slice(0, 50);
+                            oldOTPs.forEach(otp => this.processedOTPs.delete(otp));
+                        }
+                    }
+                    break;
+                }
+            }
+        } catch (err) {
+            Logger.error('Error processing log line:', {
+                error: err.message,
+                line: line.substring(0, 100)
+            });
+        }
+    }
+    
+    async sendOTPEmail(email, otpCode, logLine) {
+        try {
+            Logger.info('Processing OTP email request', {
+                email: email,
+                otpCode: otpCode.substring(0, 2) + '****', // Partial OTP for logging
+                source: logLine.substring(0, 100)
+            });
+            
+            // Determine template type based on context
+            let templateName = 'ott';
+            let subject = 'Ente - Verification Code';
+            
+            if (logLine.toLowerCase().includes('change') || logLine.toLowerCase().includes('email')) {
+                templateName = 'ott_change_email';
+                subject = 'Ente - Email Change Verification';
+            }
+            
+            // Render email template
+            const html = this.emailTemplate.render(templateName, {
+                VerificationCode: otpCode
+            });
+            
+            // Send email
+            await this.emailSender.sendEmail(email, subject, html);
+            
+            Logger.info('OTP email sent successfully', {
+                email: email,
+                template: templateName
+            });
+            
+        } catch (err) {
+            Logger.error('Failed to send OTP email:', {
+                error: err.message,
+                email: email,
+                otpCode: otpCode.substring(0, 2) + '****'
+            });
+        }
+    }
+}
+
+// Health check endpoint
+class HealthServer {
+    constructor(port = 8081) {
+        this.port = port;
+        this.server = null;
+    }
+    
+    start() {
+        const http = require('http');
+        
+        this.server = http.createServer((req, res) => {
+            if (req.url === '/health') {
+                res.writeHead(200, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify({
+                    status: 'healthy',
+                    service: 'ente-otp-email-monitor',
+                    timestamp: new Date().toISOString(),
+                    processedOTPs: monitor ? monitor.processedOTPs.size : 0
+                }));
+            } else {
+                res.writeHead(404);
+                res.end('Not Found');
+            }
+        });
+        
+        this.server.listen(this.port, () => {
+            Logger.info(`Health server listening on port ${this.port}`);
+        });
+    }
+    
+    stop() {
+        if (this.server) {
+            this.server.close();
+            Logger.info('Health server stopped');
+        }
+    }
+}
+
+// Main application
+let monitor = null;
+let healthServer = null;
+
+async function main() {
+    try {
+        Logger.info('Starting Ente OTP Email Monitor');
+        
+        // Validate environment
+        if (!process.env.CLOUDRON_EMAIL_SMTP_SERVER) {
+            throw new Error('CLOUDRON_EMAIL_SMTP_SERVER not found. Email addon may not be configured.');
+        }
+        
+        // Initialize components
+        const emailTemplate = new EmailTemplate(CONFIG.EMAIL_TEMPLATES_DIR);
+        const emailSender = new EmailSender(CONFIG);
+        
+        // Test email connectivity
+        Logger.info('Testing email connectivity...');
+        await emailSender.transporter.verify();
+        Logger.info('Email connectivity verified');
+        
+        // Start log monitor
+        monitor = new LogMonitor(CONFIG.LOG_FILE, emailSender, emailTemplate);
+        monitor.start();
+        
+        // Start health server
+        healthServer = new HealthServer();
+        healthServer.start();
+        
+        Logger.info('Ente OTP Email Monitor started successfully');
+        
+        // Handle graceful shutdown
+        process.on('SIGINT', () => {
+            Logger.info('Received SIGINT, shutting down gracefully...');
+            if (monitor) monitor.stop();
+            if (healthServer) healthServer.stop();
+            process.exit(0);
+        });
+        
+        process.on('SIGTERM', () => {
+            Logger.info('Received SIGTERM, shutting down gracefully...');
+            if (monitor) monitor.stop();
+            if (healthServer) healthServer.stop();
+            process.exit(0);
+        });
+        
+    } catch (err) {
+        Logger.error('Failed to start OTP Email Monitor:', err.message);
+        process.exit(1);
+    }
+}
+
+// Start the application
+if (require.main === module) {
+    main();
+}
+
+module.exports = {
+    LogMonitor,
+    EmailSender,
+    EmailTemplate,
+    Logger
+};

package.json

@@ -0,0 +1,22 @@
+{
+  "name": "ente-otp-email-monitor",
+  "version": "1.0.0",
+  "description": "OTP email monitoring service for Ente Cloudron app",
+  "main": "otp-email-monitor.js",
+  "scripts": {
+    "start": "node otp-email-monitor.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "nodemailer": "^6.9.0"
+  },
+  "keywords": [
+    "ente",
+    "otp",
+    "email",
+    "cloudron",
+    "monitoring"
+  ],
+  "author": "Ente Cloudron Integration",
+  "license": "Apache-2.0"
+}