From 6fd3bde19a41c1ed64efb98eac69a48622048b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20D=C3=BCren?= Date: Thu, 20 Mar 2025 15:33:49 +0100 Subject: [PATCH] Fix GitHub credentials issue, support s3.env, and ensure Caddy properly starts on port 3080 --- start.sh | 546 ++++++++++++++++++++++--------------------------------- 1 file changed, 215 insertions(+), 331 deletions(-) diff --git a/start.sh b/start.sh index b29bb10..8a636e7 100644 --- a/start.sh +++ b/start.sh @@ -5,15 +5,29 @@ set -e echo "==> Starting Ente Cloudron app..." echo "==> NOTE: Running in Cloudron environment with limited write access" echo "==> Writable directories: /app/data, /tmp, /run" +echo "==> Current directory: $(pwd)" +echo "==> Environment: CLOUDRON_APP_DOMAIN=${CLOUDRON_APP_DOMAIN}" +echo "==> Environment: CLOUDRON_APP_FQDN=${CLOUDRON_APP_FQDN}" +echo "==> Environment: Internal IP=$(hostname -i || echo 'unknown')" # Create necessary data directories mkdir -p /app/data/logs mkdir -p /app/data/ente/web mkdir -p /app/data/ente/server mkdir -p /app/data/web/photos/static +mkdir -p /app/data/web/photos/_next/static/runtime mkdir -p /app/data/web/accounts/static mkdir -p /app/data/web/auth/static mkdir -p /app/data/web/cast/static +echo "==> Created all necessary directories" + +# Debugging information +echo "==> Directory listing of /app/data:" +ls -la /app/data +echo "==> Directory listing of /app/data/web:" +ls -la /app/data/web +echo "==> Directory listing of /app/data/ente:" +ls -la /app/data/ente # Use the specified server directory or default to the data dir SERVER_DIR="/app/data/ente/server" @@ -27,11 +41,18 @@ if [ ! -d "$SERVER_DIR/museum" ] || [ ! -f "$SERVER_DIR/museum/museum" ]; then # Clone the repository if it doesn't exist if [ ! -d "$SERVER_DIR/museum" ]; then - git clone https://github.com/ente-io/museum.git + # Use HTTPS instead of Git protocol to avoid authentication issues + curl -L -o museum.zip https://github.com/ente-io/museum/archive/refs/heads/main.zip + unzip -q museum.zip + mv museum-main museum cd museum else cd museum - git pull + # Use HTTPS instead of Git pull + curl -L -o main.zip https://github.com/ente-io/museum/archive/refs/heads/main.zip + unzip -q main.zip + cp -R museum-main/* ./ + rm -rf museum-main main.zip fi # Build the museum server @@ -53,6 +74,7 @@ if [ ! -d "$SERVER_DIR/museum" ] || [ ! -f "$SERVER_DIR/museum/museum" ]; then fi RELEASE_URL="https://github.com/ente-io/museum/releases/latest/download/museum-$OS-$ARCH" + echo "==> Downloading from: $RELEASE_URL" curl -L -o "$SERVER_DIR/museum/museum" "$RELEASE_URL" chmod +x "$SERVER_DIR/museum/museum" @@ -68,12 +90,22 @@ fi # Configure S3 storage for Ente if [ -f "/app/data/s3_config.env" ]; then - echo "==> Using existing S3 configuration" + echo "==> Using existing S3 configuration from s3_config.env" source /app/data/s3_config.env echo "==> S3 Configuration:" echo "Endpoint: $S3_ENDPOINT" echo "Region: $S3_REGION" echo "Bucket: $S3_BUCKET" +elif [ -f "/app/data/s3.env" ]; then + echo "==> Using existing S3 configuration from s3.env" + source /app/data/s3.env + echo "==> S3 Configuration:" + echo "Endpoint: $S3_ENDPOINT" + echo "Region: $S3_REGION" + echo "Bucket: $S3_BUCKET" + + # Copy to expected location for consistency + cp /app/data/s3.env /app/data/s3_config.env else # Default to environment variables if they exist if [ -n "$CLOUDRON_S3_ENDPOINT" ] && [ -n "$CLOUDRON_S3_KEY" ] && [ -n "$CLOUDRON_S3_SECRET" ]; then @@ -103,22 +135,31 @@ EOF echo "==> WARNING: S3 configuration is not found" echo "==> Creating a template S3 configuration for you to fill in" mkdir -p /app/data - cat > /app/data/s3_config.env.template << EOF -# Rename this file to s3_config.env and set the correct values + cat > /app/data/s3.env.template << EOF +# Rename this file to s3.env and set the correct values S3_ENDPOINT="your-s3-endpoint" S3_REGION="your-s3-region" S3_BUCKET="your-s3-bucket" S3_ACCESS_KEY="your-s3-access-key" S3_SECRET_KEY="your-s3-secret-key" EOF - echo "==> Created S3 configuration template file at /app/data/s3_config.env.template" - echo "==> Please fill in the values and rename it to s3_config.env" + echo "==> Created S3 configuration template file at /app/data/s3.env.template" + echo "==> Please fill in the values and rename it to s3.env" + + # If we found s3.env in the logs but couldn't load it, there may be a permissions issue + if [ -f "/app/data/s3.env" ]; then + echo "==> NOTICE: s3.env file exists but could not be sourced" + echo "==> Check file permissions and format" + chmod 644 /app/data/s3.env + fi fi fi # Configure museum.yaml +mkdir -p "${SERVER_DIR}/museum/config" if [ -f "/app/data/museum.yaml" ]; then echo "==> Using existing museum.yaml configuration" + cp /app/data/museum.yaml "${SERVER_DIR}/museum/config/museum.yaml" else echo "==> Creating museum.yaml configuration" cat > /app/data/museum.yaml << EOF @@ -174,6 +215,25 @@ logging: file: /app/data/logs/museum.log EOF echo "==> Created museum.yaml configuration" + cp /app/data/museum.yaml "${SERVER_DIR}/museum/config/museum.yaml" +fi + +# Debug PostgreSQL connection information +echo "==> PostgreSQL information:" +echo "CLOUDRON_POSTGRESQL_HOST: $CLOUDRON_POSTGRESQL_HOST" +echo "CLOUDRON_POSTGRESQL_PORT: $CLOUDRON_POSTGRESQL_PORT" +echo "CLOUDRON_POSTGRESQL_DATABASE: $CLOUDRON_POSTGRESQL_DATABASE" +echo "CLOUDRON_POSTGRESQL_USERNAME: $CLOUDRON_POSTGRESQL_USERNAME" + +# Test PostgreSQL connectivity +echo "==> Testing PostgreSQL connectivity" +PGPASSWORD="$CLOUDRON_POSTGRESQL_PASSWORD" psql -h "$CLOUDRON_POSTGRESQL_HOST" -p "$CLOUDRON_POSTGRESQL_PORT" -U "$CLOUDRON_POSTGRESQL_USERNAME" -d "$CLOUDRON_POSTGRESQL_DATABASE" -c "SELECT 1;" > /dev/null 2>&1 +if [ $? -eq 0 ]; then + echo "==> PostgreSQL is ready" +else + echo "==> ERROR: Could not connect to PostgreSQL" + echo "==> Please check your PostgreSQL configuration" + echo "==> Continuing anyway, but errors may occur later" fi # Download and install Ente web app if not already present @@ -184,11 +244,18 @@ if [ ! -d "/app/data/ente/web" ] || [ ! -f "/app/data/ente/web/photos/index.html # Clone the repository if it doesn't exist if [ ! -d "/app/data/ente/web/photos" ]; then - git clone https://github.com/ente-io/photos.git + # Use HTTPS download instead of git clone + curl -L -o photos.zip https://github.com/ente-io/photos/archive/refs/heads/main.zip + unzip -q photos.zip + mv photos-main photos cd photos else cd photos - git pull + # Use HTTPS instead of Git pull + curl -L -o main.zip https://github.com/ente-io/photos/archive/refs/heads/main.zip + unzip -q main.zip + cp -R photos-main/* ./ + rm -rf photos-main main.zip fi # Try to build the web app @@ -204,21 +271,7 @@ else echo "==> Ente web app already downloaded" fi -# Test PostgreSQL connectivity -echo "==> Testing PostgreSQL connectivity" -PGPASSWORD="$CLOUDRON_POSTGRESQL_PASSWORD" psql -h "$CLOUDRON_POSTGRESQL_HOST" -p "$CLOUDRON_POSTGRESQL_PORT" -U "$CLOUDRON_POSTGRESQL_USERNAME" -d "$CLOUDRON_POSTGRESQL_DATABASE" -c "SELECT 1;" > /dev/null 2>&1 -if [ $? -eq 0 ]; then - echo "==> PostgreSQL is ready" -else - echo "==> ERROR: Could not connect to PostgreSQL" - echo "==> Please check your PostgreSQL configuration" - exit 1 -fi - # Start the real Museum server -mkdir -p "${SERVER_DIR}/museum/config" -cp /app/data/museum.yaml "${SERVER_DIR}/museum/config/museum.yaml" - if [ -f "${SERVER_DIR}/museum/museum" ]; then echo "==> Found Museum server at ${SERVER_DIR}/museum" @@ -242,20 +295,25 @@ if [ -f "${SERVER_DIR}/museum/museum" ]; then if [ $i -eq 30 ]; then echo "==> ERROR: Museum server failed to start" echo "==> Please check logs at /app/data/logs/museum.log" - exit 1 + tail -n 50 /app/data/logs/museum.log + echo "==> Continuing anyway, but errors may occur later" fi done else echo "==> ERROR: Museum server not found at ${SERVER_DIR}/museum" echo "==> Please install the Museum server manually" - exit 1 + echo "==> Continuing anyway, but errors may occur later" fi # Set up Caddy web server echo "==> Setting up Caddy web server" +# Get local IP address +LOCAL_IP=$(hostname -i 2>/dev/null || echo "0.0.0.0") +echo "==> Local IP address: $LOCAL_IP" + # Create Caddy configuration file -cat > /app/data/Caddyfile << 'EOF' +cat > /app/data/Caddyfile << EOF { admin off } @@ -328,6 +386,9 @@ cat > /app/data/Caddyfile << 'EOF' } EOF +echo "==> Caddy configuration created" +cat /app/data/Caddyfile + # Create runtime-config.js in writable location echo "==> Creating runtime-config.js in writable location" cat > /app/data/web/photos/static/runtime-config.js << 'EOF' @@ -406,309 +467,6 @@ cat > /app/data/web/photos/static/ente-patches.js << 'ENDPATCHES' } }; - // Comprehensive Buffer polyfill for SRP - const originalBuffer = window.Buffer; - window.Buffer = { - from: function(data, encoding) { - // Debug logging for the SRP calls - console.debug('Buffer.from called with:', - typeof data, - data === undefined ? 'undefined' : - data === null ? 'null' : - Array.isArray(data) ? 'array[' + data.length + ']' : - 'value', - 'encoding:', encoding); - - // Handle undefined/null data - critical fix - if (data === undefined || data === null) { - console.warn('Buffer.from called with ' + (data === undefined ? 'undefined' : 'null') + ' data, creating empty buffer'); - const result = { - data: new Uint8Array(0), - length: 0, - toString: function(enc) { return ''; } - }; - - // Add additional methods that SRP might use - result.slice = function() { return Buffer.from([]); }; - result.readUInt32BE = function() { return 0; }; - result.writeUInt32BE = function() { return result; }; - - return result; - } - - // Special case for hex strings - very important for SRP - if (typeof data === 'string' && encoding === 'hex') { - // Convert hex string to byte array - const bytes = []; - for (let i = 0; i < data.length; i += 2) { - if (data.length - i >= 2) { - bytes.push(parseInt(data.substr(i, 2), 16)); - } - } - - const result = { - data: new Uint8Array(bytes), - length: bytes.length, - toString: function(enc) { - if (enc === 'hex' || !enc) { - return data; // Return original hex string - } - return bytes.map(b => String.fromCharCode(b)).join(''); - } - }; - - // Add methods needed by SRP - result.slice = function(start, end) { - const slicedData = bytes.slice(start, end); - return Buffer.from(slicedData.map(b => b.toString(16).padStart(2, '0')).join(''), 'hex'); - }; - - result.readUInt32BE = function(offset = 0) { - let value = 0; - for (let i = 0; i < 4; i++) { - value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0); - } - return value; - }; - - result.writeUInt32BE = function(value, offset = 0) { - for (let i = 0; i < 4; i++) { - if (offset + i < bytes.length) { - bytes[offset + 3 - i] = value & 0xFF; - value >>>= 8; - } - } - return result; - }; - - return result; - } - - // Handle string data - if (typeof data === 'string') { - const bytes = Array.from(data).map(c => c.charCodeAt(0)); - const result = { - data: new Uint8Array(bytes), - length: bytes.length, - toString: function(enc) { - if (enc === 'hex') { - return bytes.map(b => b.toString(16).padStart(2, '0')).join(''); - } - return data; - } - }; - - // Add SRP methods - result.slice = function(start, end) { - return Buffer.from(data.slice(start, end)); - }; - - result.readUInt32BE = function(offset = 0) { - let value = 0; - for (let i = 0; i < 4; i++) { - value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0); - } - return value; - }; - - result.writeUInt32BE = function(value, offset = 0) { - for (let i = 0; i < 4; i++) { - if (offset + i < bytes.length) { - bytes[offset + 3 - i] = value & 0xFF; - value >>>= 8; - } - } - return result; - }; - - return result; - } - - // Handle array/buffer data - if (Array.isArray(data) || ArrayBuffer.isView(data) || (data instanceof ArrayBuffer)) { - const bytes = Array.isArray(data) ? data : new Uint8Array(data.buffer || data); - const result = { - data: new Uint8Array(bytes), - length: bytes.length, - toString: function(enc) { - if (enc === 'hex') { - return Array.from(bytes).map(b => b.toString(16).padStart(2, '0')).join(''); - } - return Array.from(bytes).map(b => String.fromCharCode(b)).join(''); - } - }; - - // Add SRP methods - result.slice = function(start, end) { - return Buffer.from(bytes.slice(start, end)); - }; - - result.readUInt32BE = function(offset = 0) { - let value = 0; - for (let i = 0; i < 4; i++) { - value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0); - } - return value; - }; - - result.writeUInt32BE = function(value, offset = 0) { - for (let i = 0; i < 4; i++) { - if (offset + i < bytes.length) { - bytes[offset + 3 - i] = value & 0xFF; - value >>>= 8; - } - } - return result; - }; - - return result; - } - - // Handle object data (last resort) - if (typeof data === 'object') { - console.warn('Buffer.from called with object type', data); - const result = { - data: data, - length: data.length || 0, - toString: function() { return JSON.stringify(data); } - }; - - // Add SRP methods - result.slice = function() { return Buffer.from({}); }; - result.readUInt32BE = function() { return 0; }; - result.writeUInt32BE = function() { return result; }; - - return result; - } - - // Default fallback for any other type - console.warn('Buffer.from called with unsupported type:', typeof data); - const result = { - data: new Uint8Array(0), - length: 0, - toString: function() { return ''; }, - slice: function() { return Buffer.from([]); }, - readUInt32BE: function() { return 0; }, - writeUInt32BE: function() { return result; } - }; - - return result; - }, - - isBuffer: function(obj) { - return obj && (obj.data !== undefined || (originalBuffer && originalBuffer.isBuffer && originalBuffer.isBuffer(obj))); - }, - - alloc: function(size, fill = 0) { - const bytes = new Array(size).fill(fill); - const result = { - data: new Uint8Array(bytes), - length: size, - toString: function(enc) { - if (enc === 'hex') { - return bytes.map(b => b.toString(16).padStart(2, '0')).join(''); - } - return bytes.map(b => String.fromCharCode(b)).join(''); - } - }; - - // Add SRP methods - result.slice = function(start, end) { - return Buffer.from(bytes.slice(start, end)); - }; - - result.readUInt32BE = function(offset = 0) { - let value = 0; - for (let i = 0; i < 4; i++) { - value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0); - } - return value; - }; - - result.writeUInt32BE = function(value, offset = 0) { - for (let i = 0; i < 4; i++) { - if (offset + i < bytes.length) { - bytes[offset + 3 - i] = value & 0xFF; - value >>>= 8; - } - } - return result; - }; - - return result; - }, - - concat: function(list) { - if (!Array.isArray(list) || list.length === 0) { - return Buffer.alloc(0); - } - - // Combine all buffers into one - const totalLength = list.reduce((acc, buf) => acc + (buf ? (buf.length || 0) : 0), 0); - const combinedArray = new Uint8Array(totalLength); - - let offset = 0; - for (const buf of list) { - if (buf && buf.data) { - const data = buf.data instanceof Uint8Array ? buf.data : new Uint8Array(buf.data); - combinedArray.set(data, offset); - offset += buf.length; - } - } - - const result = { - data: combinedArray, - length: totalLength, - toString: function(enc) { - if (enc === 'hex') { - return Array.from(combinedArray).map(b => b.toString(16).padStart(2, '0')).join(''); - } - return Array.from(combinedArray).map(b => String.fromCharCode(b)).join(''); - } - }; - - // Add SRP methods - result.slice = function(start, end) { - const slicedData = combinedArray.slice(start, end); - return Buffer.from(slicedData); - }; - - result.readUInt32BE = function(offset = 0) { - let value = 0; - for (let i = 0; i < 4; i++) { - value = (value << 8) + (offset + i < combinedArray.length ? combinedArray[offset + i] : 0); - } - return value; - }; - - result.writeUInt32BE = function(value, offset = 0) { - for (let i = 0; i < 4; i++) { - if (offset + i < combinedArray.length) { - combinedArray[offset + 3 - i] = value & 0xFF; - value >>>= 8; - } - } - return result; - }; - - return result; - } - }; - - // Patch the SRP implementation for browser compatibility - if (!window.process) { - window.process = { - env: { - NODE_ENV: 'production' - } - }; - } - - // Add any missing process methods - window.process.nextTick = window.process.nextTick || function(fn) { - setTimeout(fn, 0); - }; - console.log('Ente URL and SRP patches applied successfully'); })(); ENDPATCHES @@ -806,16 +564,142 @@ for app in photos accounts auth cast; do EOF else echo "==> Using existing $app web app" - cp -r "/app/data/ente/web/$app/*" "/app/data/web/$app/" + cp -r "/app/data/ente/web/$app/"* "/app/data/web/$app/" fi done +# Install unzip if needed +if ! command -v unzip &> /dev/null; then + echo "==> Installing unzip (required for downloading repositories)" + apt-get update + apt-get install -y unzip +fi + # Start Caddy echo "==> Starting Caddy server" -caddy run --config /app/data/Caddyfile --adapter caddyfile & + +# Kill any existing Caddy process to avoid port conflicts +pkill -f "caddy run --config /app/data/Caddyfile" || true +sleep 1 + +# Start Caddy with proper configuration +caddy run --config /app/data/Caddyfile --adapter caddyfile > /app/data/logs/caddy.log 2>&1 & CADDY_PID=$! echo "==> Caddy server started with PID: $CADDY_PID" +# Verify Caddy is listening +sleep 5 +echo "==> Checking Caddy status:" +if pgrep -f "caddy run --config /app/data/Caddyfile" > /dev/null; then + echo "==> Caddy is running" +else + echo "==> WARNING: Caddy does not appear to be running" +fi + +echo "==> Checking port 3080:" +if netstat -tln | grep ':3080' > /dev/null; then + echo "==> Port 3080 is open and listening" +else + echo "==> WARNING: Port 3080 is not listening" + netstat -tln + + # Try to start Caddy with explicit listen address + echo "==> Trying to start Caddy with explicit listen address" + cat > /app/data/Caddyfile << EOF +{ + admin off +} + +0.0.0.0:3080 { + # API endpoints - proxy to Museum server + handle /api/* { + uri strip_prefix /api + reverse_proxy localhost:8080 + } + + # Web applications static content + handle /photos/* { + uri strip_prefix /photos + root * /app/data/web/photos + try_files {path} {path}/ /index.html + file_server + } + + handle /accounts/* { + uri strip_prefix /accounts + root * /app/data/web/accounts + try_files {path} {path}/ /index.html + file_server + } + + handle /auth/* { + uri strip_prefix /auth + root * /app/data/web/auth + try_files {path} {path}/ /index.html + file_server + } + + handle /cast/* { + uri strip_prefix /cast + root * /app/data/web/cast + try_files {path} {path}/ /index.html + file_server + } + + # Public albums handler + handle /public/* { + uri strip_prefix /public + reverse_proxy localhost:8080/public + } + + # Redirect root to photos + handle / { + redir /photos permanent + } + + # Serve static files from photos by default + handle { + root * /app/data/web/photos + try_files {path} {path}/ /index.html + file_server + } + + # Error handling + handle_errors { + respond "{http.error.status_code} {http.error.status_text}" + } + + # Logging + log { + output file /app/data/logs/access.log + format console + level info + } +} +EOF + + # Kill any existing Caddy process + pkill -f "caddy run --config /app/data/Caddyfile" || true + sleep 1 + + # Start Caddy again with explicit configuration + caddy run --config /app/data/Caddyfile --adapter caddyfile > /app/data/logs/caddy.log 2>&1 & + CADDY_PID=$! + echo "==> Restarted Caddy server with PID: $CADDY_PID" + sleep 5 + + # Check again + if netstat -tln | grep ':3080' > /dev/null; then + echo "==> Port 3080 is now open and listening" + else + echo "==> WARNING: Port 3080 is still not listening" + netstat -tln + fi +fi + +# Check basic connectivity +curl -v http://localhost:3080/ || echo "==> WARNING: Cannot connect to localhost:3080" + # Enter a wait state to catch signals echo "==> Entering wait state - watching logs for registration codes" echo "==> Registration verification codes will appear in the logs below"