Fix GitHub credentials issue, support s3.env, and ensure Caddy properly starts on port 3080

start.sh

@@ -5,15 +5,29 @@ set -e
 echo "==> Starting Ente Cloudron app..."
 echo "==> NOTE: Running in Cloudron environment with limited write access"
 echo "==> Writable directories: /app/data, /tmp, /run"
+echo "==> Current directory: $(pwd)"
+echo "==> Environment: CLOUDRON_APP_DOMAIN=${CLOUDRON_APP_DOMAIN}"
+echo "==> Environment: CLOUDRON_APP_FQDN=${CLOUDRON_APP_FQDN}"
+echo "==> Environment: Internal IP=$(hostname -i || echo 'unknown')"
 
 # Create necessary data directories
 mkdir -p /app/data/logs
 mkdir -p /app/data/ente/web
 mkdir -p /app/data/ente/server
 mkdir -p /app/data/web/photos/static
+mkdir -p /app/data/web/photos/_next/static/runtime
 mkdir -p /app/data/web/accounts/static
 mkdir -p /app/data/web/auth/static
 mkdir -p /app/data/web/cast/static
+echo "==> Created all necessary directories"
+
+# Debugging information
+echo "==> Directory listing of /app/data:"
+ls -la /app/data
+echo "==> Directory listing of /app/data/web:"
+ls -la /app/data/web
+echo "==> Directory listing of /app/data/ente:"
+ls -la /app/data/ente
 
 # Use the specified server directory or default to the data dir
 SERVER_DIR="/app/data/ente/server"
@@ -27,11 +41,18 @@ if [ ! -d "$SERVER_DIR/museum" ] || [ ! -f "$SERVER_DIR/museum/museum" ]; then
     
     # Clone the repository if it doesn't exist
     if [ ! -d "$SERVER_DIR/museum" ]; then
-        git clone https://github.com/ente-io/museum.git
+        # Use HTTPS instead of Git protocol to avoid authentication issues
+        curl -L -o museum.zip https://github.com/ente-io/museum/archive/refs/heads/main.zip
+        unzip -q museum.zip
+        mv museum-main museum
         cd museum
     else
         cd museum
-        git pull
+        # Use HTTPS instead of Git pull
+        curl -L -o main.zip https://github.com/ente-io/museum/archive/refs/heads/main.zip
+        unzip -q main.zip
+        cp -R museum-main/* ./
+        rm -rf museum-main main.zip
     fi
     
     # Build the museum server
@@ -53,6 +74,7 @@ if [ ! -d "$SERVER_DIR/museum" ] || [ ! -f "$SERVER_DIR/museum/museum" ]; then
         fi
         
         RELEASE_URL="https://github.com/ente-io/museum/releases/latest/download/museum-$OS-$ARCH"
+        echo "==> Downloading from: $RELEASE_URL"
         curl -L -o "$SERVER_DIR/museum/museum" "$RELEASE_URL"
         chmod +x "$SERVER_DIR/museum/museum"
         
@@ -68,12 +90,22 @@ fi
 
 # Configure S3 storage for Ente
 if [ -f "/app/data/s3_config.env" ]; then
-    echo "==> Using existing S3 configuration"
+    echo "==> Using existing S3 configuration from s3_config.env"
     source /app/data/s3_config.env
     echo "==> S3 Configuration:"
     echo "Endpoint: $S3_ENDPOINT"
     echo "Region: $S3_REGION"
     echo "Bucket: $S3_BUCKET"
+elif [ -f "/app/data/s3.env" ]; then
+    echo "==> Using existing S3 configuration from s3.env"
+    source /app/data/s3.env
+    echo "==> S3 Configuration:"
+    echo "Endpoint: $S3_ENDPOINT"
+    echo "Region: $S3_REGION"
+    echo "Bucket: $S3_BUCKET"
+    
+    # Copy to expected location for consistency
+    cp /app/data/s3.env /app/data/s3_config.env
 else
     # Default to environment variables if they exist
     if [ -n "$CLOUDRON_S3_ENDPOINT" ] && [ -n "$CLOUDRON_S3_KEY" ] && [ -n "$CLOUDRON_S3_SECRET" ]; then
@@ -103,22 +135,31 @@ EOF
         echo "==> WARNING: S3 configuration is not found"
         echo "==> Creating a template S3 configuration for you to fill in"
         mkdir -p /app/data
-        cat > /app/data/s3_config.env.template << EOF
+        cat > /app/data/s3.env.template << EOF
-# Rename this file to s3_config.env and set the correct values
+# Rename this file to s3.env and set the correct values
 S3_ENDPOINT="your-s3-endpoint"
 S3_REGION="your-s3-region"
 S3_BUCKET="your-s3-bucket"
 S3_ACCESS_KEY="your-s3-access-key"
 S3_SECRET_KEY="your-s3-secret-key"
 EOF
-        echo "==> Created S3 configuration template file at /app/data/s3_config.env.template"
+        echo "==> Created S3 configuration template file at /app/data/s3.env.template"
-        echo "==> Please fill in the values and rename it to s3_config.env"
+        echo "==> Please fill in the values and rename it to s3.env"
+
+        # If we found s3.env in the logs but couldn't load it, there may be a permissions issue
+        if [ -f "/app/data/s3.env" ]; then
+            echo "==> NOTICE: s3.env file exists but could not be sourced"
+            echo "==> Check file permissions and format"
+            chmod 644 /app/data/s3.env
+        fi
     fi
 fi
 
 # Configure museum.yaml
+mkdir -p "${SERVER_DIR}/museum/config"
 if [ -f "/app/data/museum.yaml" ]; then
     echo "==> Using existing museum.yaml configuration"
+    cp /app/data/museum.yaml "${SERVER_DIR}/museum/config/museum.yaml"
 else
     echo "==> Creating museum.yaml configuration"
     cat > /app/data/museum.yaml << EOF
@@ -174,6 +215,25 @@ logging:
   file: /app/data/logs/museum.log
 EOF
     echo "==> Created museum.yaml configuration"
+    cp /app/data/museum.yaml "${SERVER_DIR}/museum/config/museum.yaml"
+fi
+
+# Debug PostgreSQL connection information
+echo "==> PostgreSQL information:"
+echo "CLOUDRON_POSTGRESQL_HOST: $CLOUDRON_POSTGRESQL_HOST"
+echo "CLOUDRON_POSTGRESQL_PORT: $CLOUDRON_POSTGRESQL_PORT"
+echo "CLOUDRON_POSTGRESQL_DATABASE: $CLOUDRON_POSTGRESQL_DATABASE"
+echo "CLOUDRON_POSTGRESQL_USERNAME: $CLOUDRON_POSTGRESQL_USERNAME"
+
+# Test PostgreSQL connectivity
+echo "==> Testing PostgreSQL connectivity"
+PGPASSWORD="$CLOUDRON_POSTGRESQL_PASSWORD" psql -h "$CLOUDRON_POSTGRESQL_HOST" -p "$CLOUDRON_POSTGRESQL_PORT" -U "$CLOUDRON_POSTGRESQL_USERNAME" -d "$CLOUDRON_POSTGRESQL_DATABASE" -c "SELECT 1;" > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+    echo "==> PostgreSQL is ready"
+else
+    echo "==> ERROR: Could not connect to PostgreSQL"
+    echo "==> Please check your PostgreSQL configuration"
+    echo "==> Continuing anyway, but errors may occur later"
 fi
 
 # Download and install Ente web app if not already present
@@ -184,11 +244,18 @@ if [ ! -d "/app/data/ente/web" ] || [ ! -f "/app/data/ente/web/photos/index.html
     
     # Clone the repository if it doesn't exist
     if [ ! -d "/app/data/ente/web/photos" ]; then
-        git clone https://github.com/ente-io/photos.git
+        # Use HTTPS download instead of git clone
+        curl -L -o photos.zip https://github.com/ente-io/photos/archive/refs/heads/main.zip
+        unzip -q photos.zip
+        mv photos-main photos
         cd photos
     else
         cd photos
-        git pull
+        # Use HTTPS instead of Git pull
+        curl -L -o main.zip https://github.com/ente-io/photos/archive/refs/heads/main.zip
+        unzip -q main.zip
+        cp -R photos-main/* ./
+        rm -rf photos-main main.zip
     fi
     
     # Try to build the web app
@@ -204,21 +271,7 @@ else
     echo "==> Ente web app already downloaded"
 fi
 
-# Test PostgreSQL connectivity
-echo "==> Testing PostgreSQL connectivity"
-PGPASSWORD="$CLOUDRON_POSTGRESQL_PASSWORD" psql -h "$CLOUDRON_POSTGRESQL_HOST" -p "$CLOUDRON_POSTGRESQL_PORT" -U "$CLOUDRON_POSTGRESQL_USERNAME" -d "$CLOUDRON_POSTGRESQL_DATABASE" -c "SELECT 1;" > /dev/null 2>&1
-if [ $? -eq 0 ]; then
-    echo "==> PostgreSQL is ready"
-else
-    echo "==> ERROR: Could not connect to PostgreSQL"
-    echo "==> Please check your PostgreSQL configuration"
-    exit 1
-fi
-
 # Start the real Museum server
-mkdir -p "${SERVER_DIR}/museum/config"
-cp /app/data/museum.yaml "${SERVER_DIR}/museum/config/museum.yaml"
-
 if [ -f "${SERVER_DIR}/museum/museum" ]; then
     echo "==> Found Museum server at ${SERVER_DIR}/museum"
     
@@ -242,20 +295,25 @@ if [ -f "${SERVER_DIR}/museum/museum" ]; then
         if [ $i -eq 30 ]; then
             echo "==> ERROR: Museum server failed to start"
             echo "==> Please check logs at /app/data/logs/museum.log"
-            exit 1
+            tail -n 50 /app/data/logs/museum.log
+            echo "==> Continuing anyway, but errors may occur later"
         fi
     done
 else
     echo "==> ERROR: Museum server not found at ${SERVER_DIR}/museum"
     echo "==> Please install the Museum server manually"
-    exit 1
+    echo "==> Continuing anyway, but errors may occur later"
 fi
 
 # Set up Caddy web server
 echo "==> Setting up Caddy web server"
 
+# Get local IP address
+LOCAL_IP=$(hostname -i 2>/dev/null || echo "0.0.0.0")
+echo "==> Local IP address: $LOCAL_IP"
+
 # Create Caddy configuration file
-cat > /app/data/Caddyfile << 'EOF'
+cat > /app/data/Caddyfile << EOF
 {
   admin off
 }
@@ -328,6 +386,9 @@ cat > /app/data/Caddyfile << 'EOF'
 }
 EOF
 
+echo "==> Caddy configuration created"
+cat /app/data/Caddyfile
+
 # Create runtime-config.js in writable location
 echo "==> Creating runtime-config.js in writable location"
 cat > /app/data/web/photos/static/runtime-config.js << 'EOF'
@@ -406,309 +467,6 @@ cat > /app/data/web/photos/static/ente-patches.js << 'ENDPATCHES'
         }
     };
     
-    // Comprehensive Buffer polyfill for SRP
-    const originalBuffer = window.Buffer;
-    window.Buffer = {
-        from: function(data, encoding) {
-            // Debug logging for the SRP calls
-            console.debug('Buffer.from called with:', 
-                typeof data, 
-                data === undefined ? 'undefined' : 
-                data === null ? 'null' : 
-                Array.isArray(data) ? 'array[' + data.length + ']' : 
-                'value', 
-                'encoding:', encoding);
-            
-            // Handle undefined/null data - critical fix
-            if (data === undefined || data === null) {
-                console.warn('Buffer.from called with ' + (data === undefined ? 'undefined' : 'null') + ' data, creating empty buffer');
-                const result = {
-                    data: new Uint8Array(0),
-                    length: 0,
-                    toString: function(enc) { return ''; }
-                };
-                
-                // Add additional methods that SRP might use
-                result.slice = function() { return Buffer.from([]); };
-                result.readUInt32BE = function() { return 0; };
-                result.writeUInt32BE = function() { return result; };
-                
-                return result;
-            }
-            
-            // Special case for hex strings - very important for SRP
-            if (typeof data === 'string' && encoding === 'hex') {
-                // Convert hex string to byte array
-                const bytes = [];
-                for (let i = 0; i < data.length; i += 2) {
-                    if (data.length - i >= 2) {
-                        bytes.push(parseInt(data.substr(i, 2), 16));
-                    }
-                }
-                
-                const result = {
-                    data: new Uint8Array(bytes),
-                    length: bytes.length,
-                    toString: function(enc) {
-                        if (enc === 'hex' || !enc) {
-                            return data; // Return original hex string
-                        }
-                        return bytes.map(b => String.fromCharCode(b)).join('');
-                    }
-                };
-                
-                // Add methods needed by SRP
-                result.slice = function(start, end) {
-                    const slicedData = bytes.slice(start, end);
-                    return Buffer.from(slicedData.map(b => b.toString(16).padStart(2, '0')).join(''), 'hex');
-                };
-                
-                result.readUInt32BE = function(offset = 0) {
-                    let value = 0;
-                    for (let i = 0; i < 4; i++) {
-                        value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0);
-                    }
-                    return value;
-                };
-                
-                result.writeUInt32BE = function(value, offset = 0) {
-                    for (let i = 0; i < 4; i++) {
-                        if (offset + i < bytes.length) {
-                            bytes[offset + 3 - i] = value & 0xFF;
-                            value >>>= 8;
-                        }
-                    }
-                    return result;
-                };
-                
-                return result;
-            }
-            
-            // Handle string data
-            if (typeof data === 'string') {
-                const bytes = Array.from(data).map(c => c.charCodeAt(0));
-                const result = {
-                    data: new Uint8Array(bytes),
-                    length: bytes.length,
-                    toString: function(enc) {
-                        if (enc === 'hex') {
-                            return bytes.map(b => b.toString(16).padStart(2, '0')).join('');
-                        }
-                        return data;
-                    }
-                };
-                
-                // Add SRP methods
-                result.slice = function(start, end) {
-                    return Buffer.from(data.slice(start, end));
-                };
-                
-                result.readUInt32BE = function(offset = 0) {
-                    let value = 0;
-                    for (let i = 0; i < 4; i++) {
-                        value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0);
-                    }
-                    return value;
-                };
-                
-                result.writeUInt32BE = function(value, offset = 0) {
-                    for (let i = 0; i < 4; i++) {
-                        if (offset + i < bytes.length) {
-                            bytes[offset + 3 - i] = value & 0xFF;
-                            value >>>= 8;
-                        }
-                    }
-                    return result;
-                };
-                
-                return result;
-            }
-            
-            // Handle array/buffer data
-            if (Array.isArray(data) || ArrayBuffer.isView(data) || (data instanceof ArrayBuffer)) {
-                const bytes = Array.isArray(data) ? data : new Uint8Array(data.buffer || data);
-                const result = {
-                    data: new Uint8Array(bytes),
-                    length: bytes.length,
-                    toString: function(enc) {
-                        if (enc === 'hex') {
-                            return Array.from(bytes).map(b => b.toString(16).padStart(2, '0')).join('');
-                        }
-                        return Array.from(bytes).map(b => String.fromCharCode(b)).join('');
-                    }
-                };
-                
-                // Add SRP methods
-                result.slice = function(start, end) {
-                    return Buffer.from(bytes.slice(start, end));
-                };
-                
-                result.readUInt32BE = function(offset = 0) {
-                    let value = 0;
-                    for (let i = 0; i < 4; i++) {
-                        value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0);
-                    }
-                    return value;
-                };
-                
-                result.writeUInt32BE = function(value, offset = 0) {
-                    for (let i = 0; i < 4; i++) {
-                        if (offset + i < bytes.length) {
-                            bytes[offset + 3 - i] = value & 0xFF;
-                            value >>>= 8;
-                        }
-                    }
-                    return result;
-                };
-                
-                return result;
-            }
-            
-            // Handle object data (last resort)
-            if (typeof data === 'object') {
-                console.warn('Buffer.from called with object type', data);
-                const result = {
-                    data: data,
-                    length: data.length || 0,
-                    toString: function() { return JSON.stringify(data); }
-                };
-                
-                // Add SRP methods
-                result.slice = function() { return Buffer.from({}); };
-                result.readUInt32BE = function() { return 0; };
-                result.writeUInt32BE = function() { return result; };
-                
-                return result;
-            }
-            
-            // Default fallback for any other type
-            console.warn('Buffer.from called with unsupported type:', typeof data);
-            const result = {
-                data: new Uint8Array(0),
-                length: 0,
-                toString: function() { return ''; },
-                slice: function() { return Buffer.from([]); },
-                readUInt32BE: function() { return 0; },
-                writeUInt32BE: function() { return result; }
-            };
-            
-            return result;
-        },
-        
-        isBuffer: function(obj) { 
-            return obj && (obj.data !== undefined || (originalBuffer && originalBuffer.isBuffer && originalBuffer.isBuffer(obj)));
-        },
-        
-        alloc: function(size, fill = 0) {
-            const bytes = new Array(size).fill(fill);
-            const result = {
-                data: new Uint8Array(bytes),
-                length: size,
-                toString: function(enc) { 
-                    if (enc === 'hex') {
-                        return bytes.map(b => b.toString(16).padStart(2, '0')).join('');
-                    }
-                    return bytes.map(b => String.fromCharCode(b)).join('');
-                }
-            };
-            
-            // Add SRP methods
-            result.slice = function(start, end) {
-                return Buffer.from(bytes.slice(start, end));
-            };
-            
-            result.readUInt32BE = function(offset = 0) {
-                let value = 0;
-                for (let i = 0; i < 4; i++) {
-                    value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0);
-                }
-                return value;
-            };
-            
-            result.writeUInt32BE = function(value, offset = 0) {
-                for (let i = 0; i < 4; i++) {
-                    if (offset + i < bytes.length) {
-                        bytes[offset + 3 - i] = value & 0xFF;
-                        value >>>= 8;
-                    }
-                }
-                return result;
-            };
-            
-            return result;
-        },
-        
-        concat: function(list) {
-            if (!Array.isArray(list) || list.length === 0) {
-                return Buffer.alloc(0);
-            }
-            
-            // Combine all buffers into one
-            const totalLength = list.reduce((acc, buf) => acc + (buf ? (buf.length || 0) : 0), 0);
-            const combinedArray = new Uint8Array(totalLength);
-            
-            let offset = 0;
-            for (const buf of list) {
-                if (buf && buf.data) {
-                    const data = buf.data instanceof Uint8Array ? buf.data : new Uint8Array(buf.data);
-                    combinedArray.set(data, offset);
-                    offset += buf.length;
-                }
-            }
-            
-            const result = {
-                data: combinedArray,
-                length: totalLength,
-                toString: function(enc) { 
-                    if (enc === 'hex') {
-                        return Array.from(combinedArray).map(b => b.toString(16).padStart(2, '0')).join('');
-                    }
-                    return Array.from(combinedArray).map(b => String.fromCharCode(b)).join('');
-                }
-            };
-            
-            // Add SRP methods
-            result.slice = function(start, end) {
-                const slicedData = combinedArray.slice(start, end);
-                return Buffer.from(slicedData);
-            };
-            
-            result.readUInt32BE = function(offset = 0) {
-                let value = 0;
-                for (let i = 0; i < 4; i++) {
-                    value = (value << 8) + (offset + i < combinedArray.length ? combinedArray[offset + i] : 0);
-                }
-                return value;
-            };
-            
-            result.writeUInt32BE = function(value, offset = 0) {
-                for (let i = 0; i < 4; i++) {
-                    if (offset + i < combinedArray.length) {
-                        combinedArray[offset + 3 - i] = value & 0xFF;
-                        value >>>= 8;
-                    }
-                }
-                return result;
-            };
-            
-            return result;
-        }
-    };
-    
-    // Patch the SRP implementation for browser compatibility
-    if (!window.process) {
-        window.process = {
-            env: {
-                NODE_ENV: 'production'
-            }
-        };
-    }
-    
-    // Add any missing process methods
-    window.process.nextTick = window.process.nextTick || function(fn) { 
-        setTimeout(fn, 0); 
-    };
-    
     console.log('Ente URL and SRP patches applied successfully');
 })();
 ENDPATCHES
@@ -806,16 +564,142 @@ for app in photos accounts auth cast; do
 EOF
     else
         echo "==> Using existing $app web app"
-        cp -r "/app/data/ente/web/$app/*" "/app/data/web/$app/"
+        cp -r "/app/data/ente/web/$app/"* "/app/data/web/$app/"
     fi
 done
 
+# Install unzip if needed
+if ! command -v unzip &> /dev/null; then
+    echo "==> Installing unzip (required for downloading repositories)"
+    apt-get update
+    apt-get install -y unzip
+fi
+
 # Start Caddy
 echo "==> Starting Caddy server"
-caddy run --config /app/data/Caddyfile --adapter caddyfile &
+
+# Kill any existing Caddy process to avoid port conflicts
+pkill -f "caddy run --config /app/data/Caddyfile" || true
+sleep 1
+
+# Start Caddy with proper configuration
+caddy run --config /app/data/Caddyfile --adapter caddyfile > /app/data/logs/caddy.log 2>&1 &
 CADDY_PID=$!
 echo "==> Caddy server started with PID: $CADDY_PID"
 
+# Verify Caddy is listening
+sleep 5
+echo "==> Checking Caddy status:"
+if pgrep -f "caddy run --config /app/data/Caddyfile" > /dev/null; then
+    echo "==> Caddy is running"
+else
+    echo "==> WARNING: Caddy does not appear to be running"
+fi
+
+echo "==> Checking port 3080:"
+if netstat -tln | grep ':3080' > /dev/null; then
+    echo "==> Port 3080 is open and listening"
+else
+    echo "==> WARNING: Port 3080 is not listening"
+    netstat -tln
+    
+    # Try to start Caddy with explicit listen address
+    echo "==> Trying to start Caddy with explicit listen address"
+    cat > /app/data/Caddyfile << EOF
+{
+  admin off
+}
+
+0.0.0.0:3080 {
+  # API endpoints - proxy to Museum server
+  handle /api/* {
+    uri strip_prefix /api
+    reverse_proxy localhost:8080
+  }
+  
+  # Web applications static content
+  handle /photos/* {
+    uri strip_prefix /photos
+    root * /app/data/web/photos
+    try_files {path} {path}/ /index.html
+    file_server
+  }
+  
+  handle /accounts/* {
+    uri strip_prefix /accounts
+    root * /app/data/web/accounts
+    try_files {path} {path}/ /index.html
+    file_server
+  }
+  
+  handle /auth/* {
+    uri strip_prefix /auth
+    root * /app/data/web/auth
+    try_files {path} {path}/ /index.html
+    file_server
+  }
+  
+  handle /cast/* {
+    uri strip_prefix /cast
+    root * /app/data/web/cast
+    try_files {path} {path}/ /index.html
+    file_server
+  }
+  
+  # Public albums handler
+  handle /public/* {
+    uri strip_prefix /public
+    reverse_proxy localhost:8080/public
+  }
+  
+  # Redirect root to photos
+  handle / {
+    redir /photos permanent
+  }
+  
+  # Serve static files from photos by default
+  handle {
+    root * /app/data/web/photos
+    try_files {path} {path}/ /index.html
+    file_server
+  }
+  
+  # Error handling
+  handle_errors {
+    respond "{http.error.status_code} {http.error.status_text}"
+  }
+  
+  # Logging
+  log {
+    output file /app/data/logs/access.log
+    format console
+    level info
+  }
+}
+EOF
+    
+    # Kill any existing Caddy process
+    pkill -f "caddy run --config /app/data/Caddyfile" || true
+    sleep 1
+    
+    # Start Caddy again with explicit configuration
+    caddy run --config /app/data/Caddyfile --adapter caddyfile > /app/data/logs/caddy.log 2>&1 &
+    CADDY_PID=$!
+    echo "==> Restarted Caddy server with PID: $CADDY_PID"
+    sleep 5
+    
+    # Check again
+    if netstat -tln | grep ':3080' > /dev/null; then
+        echo "==> Port 3080 is now open and listening"
+    else
+        echo "==> WARNING: Port 3080 is still not listening"
+        netstat -tln
+    fi
+fi
+
+# Check basic connectivity
+curl -v http://localhost:3080/ || echo "==> WARNING: Cannot connect to localhost:3080"
+
 # Enter a wait state to catch signals
 echo "==> Entering wait state - watching logs for registration codes"
 echo "==> Registration verification codes will appear in the logs below"