Fix GitHub credentials issue, support s3.env, and ensure Caddy properly starts on port 3080 (6fd3bde1) · Commits · Andreas Düren / ente-cloudron

start.sh

+215 −331

Original line number	Diff line number	Diff line
		@@ -5,15 +5,29 @@ set -e
		echo "==> Starting Ente Cloudron app..."
		echo "==> NOTE: Running in Cloudron environment with limited write access"
		echo "==> Writable directories: /app/data, /tmp, /run"
		echo "==> Current directory: $(pwd)"
		echo "==> Environment: CLOUDRON_APP_DOMAIN=${CLOUDRON_APP_DOMAIN}"
		echo "==> Environment: CLOUDRON_APP_FQDN=${CLOUDRON_APP_FQDN}"
		echo "==> Environment: Internal IP=$(hostname -i \|\| echo 'unknown')"

		# Create necessary data directories
		mkdir -p /app/data/logs
		mkdir -p /app/data/ente/web
		mkdir -p /app/data/ente/server
		mkdir -p /app/data/web/photos/static
		mkdir -p /app/data/web/photos/_next/static/runtime
		mkdir -p /app/data/web/accounts/static
		mkdir -p /app/data/web/auth/static
		mkdir -p /app/data/web/cast/static
		echo "==> Created all necessary directories"

		# Debugging information
		echo "==> Directory listing of /app/data:"
		ls -la /app/data
		echo "==> Directory listing of /app/data/web:"
		ls -la /app/data/web
		echo "==> Directory listing of /app/data/ente:"
		ls -la /app/data/ente

		# Use the specified server directory or default to the data dir
		SERVER_DIR="/app/data/ente/server"
		@@ -27,11 +41,18 @@ if [ ! -d "$SERVER_DIR/museum" ] \|\| [ ! -f "$SERVER_DIR/museum/museum" ]; then

		# Clone the repository if it doesn't exist
		if [ ! -d "$SERVER_DIR/museum" ]; then
		git clone https://github.com/ente-io/museum.git
		# Use HTTPS instead of Git protocol to avoid authentication issues
		curl -L -o museum.zip https://github.com/ente-io/museum/archive/refs/heads/main.zip
		unzip -q museum.zip
		mv museum-main museum
		cd museum
		else
		cd museum
		git pull
		# Use HTTPS instead of Git pull
		curl -L -o main.zip https://github.com/ente-io/museum/archive/refs/heads/main.zip
		unzip -q main.zip
		cp -R museum-main/* ./
		rm -rf museum-main main.zip
		fi

		# Build the museum server
		@@ -53,6 +74,7 @@ if [ ! -d "$SERVER_DIR/museum" ] \|\| [ ! -f "$SERVER_DIR/museum/museum" ]; then
		fi

		RELEASE_URL="https://github.com/ente-io/museum/releases/latest/download/museum-$OS-$ARCH"
		echo "==> Downloading from: $RELEASE_URL"
		curl -L -o "$SERVER_DIR/museum/museum" "$RELEASE_URL"
		chmod +x "$SERVER_DIR/museum/museum"

		@@ -68,12 +90,22 @@ fi

		# Configure S3 storage for Ente
		if [ -f "/app/data/s3_config.env" ]; then
		echo "==> Using existing S3 configuration"
		echo "==> Using existing S3 configuration from s3_config.env"
		source /app/data/s3_config.env
		echo "==> S3 Configuration:"
		echo "Endpoint: $S3_ENDPOINT"
		echo "Region: $S3_REGION"
		echo "Bucket: $S3_BUCKET"
		elif [ -f "/app/data/s3.env" ]; then
		echo "==> Using existing S3 configuration from s3.env"
		source /app/data/s3.env
		echo "==> S3 Configuration:"
		echo "Endpoint: $S3_ENDPOINT"
		echo "Region: $S3_REGION"
		echo "Bucket: $S3_BUCKET"

		# Copy to expected location for consistency
		cp /app/data/s3.env /app/data/s3_config.env
		else
		# Default to environment variables if they exist
		if [ -n "$CLOUDRON_S3_ENDPOINT" ] && [ -n "$CLOUDRON_S3_KEY" ] && [ -n "$CLOUDRON_S3_SECRET" ]; then
		@@ -103,22 +135,31 @@ EOF
		echo "==> WARNING: S3 configuration is not found"
		echo "==> Creating a template S3 configuration for you to fill in"
		mkdir -p /app/data
		cat > /app/data/s3_config.env.template << EOF
		# Rename this file to s3_config.env and set the correct values
		cat > /app/data/s3.env.template << EOF
		# Rename this file to s3.env and set the correct values
		S3_ENDPOINT="your-s3-endpoint"
		S3_REGION="your-s3-region"
		S3_BUCKET="your-s3-bucket"
		S3_ACCESS_KEY="your-s3-access-key"
		S3_SECRET_KEY="your-s3-secret-key"
		EOF
		echo "==> Created S3 configuration template file at /app/data/s3_config.env.template"
		echo "==> Please fill in the values and rename it to s3_config.env"
		echo "==> Created S3 configuration template file at /app/data/s3.env.template"
		echo "==> Please fill in the values and rename it to s3.env"

		# If we found s3.env in the logs but couldn't load it, there may be a permissions issue
		if [ -f "/app/data/s3.env" ]; then
		echo "==> NOTICE: s3.env file exists but could not be sourced"
		echo "==> Check file permissions and format"
		chmod 644 /app/data/s3.env
		fi
		fi
		fi

		# Configure museum.yaml
		mkdir -p "${SERVER_DIR}/museum/config"
		if [ -f "/app/data/museum.yaml" ]; then
		echo "==> Using existing museum.yaml configuration"
		cp /app/data/museum.yaml "${SERVER_DIR}/museum/config/museum.yaml"
		else
		echo "==> Creating museum.yaml configuration"
		cat > /app/data/museum.yaml << EOF
		@@ -174,6 +215,25 @@ logging:
		file: /app/data/logs/museum.log
		EOF
		echo "==> Created museum.yaml configuration"
		cp /app/data/museum.yaml "${SERVER_DIR}/museum/config/museum.yaml"
		fi

		# Debug PostgreSQL connection information
		echo "==> PostgreSQL information:"
		echo "CLOUDRON_POSTGRESQL_HOST: $CLOUDRON_POSTGRESQL_HOST"
		echo "CLOUDRON_POSTGRESQL_PORT: $CLOUDRON_POSTGRESQL_PORT"
		echo "CLOUDRON_POSTGRESQL_DATABASE: $CLOUDRON_POSTGRESQL_DATABASE"
		echo "CLOUDRON_POSTGRESQL_USERNAME: $CLOUDRON_POSTGRESQL_USERNAME"

		# Test PostgreSQL connectivity
		echo "==> Testing PostgreSQL connectivity"
		PGPASSWORD="$CLOUDRON_POSTGRESQL_PASSWORD" psql -h "$CLOUDRON_POSTGRESQL_HOST" -p "$CLOUDRON_POSTGRESQL_PORT" -U "$CLOUDRON_POSTGRESQL_USERNAME" -d "$CLOUDRON_POSTGRESQL_DATABASE" -c "SELECT 1;" > /dev/null 2>&1
		if [ $? -eq 0 ]; then
		echo "==> PostgreSQL is ready"
		else
		echo "==> ERROR: Could not connect to PostgreSQL"
		echo "==> Please check your PostgreSQL configuration"
		echo "==> Continuing anyway, but errors may occur later"
		fi

		# Download and install Ente web app if not already present
		@@ -184,11 +244,18 @@ if [ ! -d "/app/data/ente/web" ] \|\| [ ! -f "/app/data/ente/web/photos/index.html

		# Clone the repository if it doesn't exist
		if [ ! -d "/app/data/ente/web/photos" ]; then
		git clone https://github.com/ente-io/photos.git
		# Use HTTPS download instead of git clone
		curl -L -o photos.zip https://github.com/ente-io/photos/archive/refs/heads/main.zip
		unzip -q photos.zip
		mv photos-main photos
		cd photos
		else
		cd photos
		git pull
		# Use HTTPS instead of Git pull
		curl -L -o main.zip https://github.com/ente-io/photos/archive/refs/heads/main.zip
		unzip -q main.zip
		cp -R photos-main/* ./
		rm -rf photos-main main.zip
		fi

		# Try to build the web app
		@@ -204,21 +271,7 @@ else
		echo "==> Ente web app already downloaded"
		fi

		# Test PostgreSQL connectivity
		echo "==> Testing PostgreSQL connectivity"
		PGPASSWORD="$CLOUDRON_POSTGRESQL_PASSWORD" psql -h "$CLOUDRON_POSTGRESQL_HOST" -p "$CLOUDRON_POSTGRESQL_PORT" -U "$CLOUDRON_POSTGRESQL_USERNAME" -d "$CLOUDRON_POSTGRESQL_DATABASE" -c "SELECT 1;" > /dev/null 2>&1
		if [ $? -eq 0 ]; then
		echo "==> PostgreSQL is ready"
		else
		echo "==> ERROR: Could not connect to PostgreSQL"
		echo "==> Please check your PostgreSQL configuration"
		exit 1
		fi

		# Start the real Museum server
		mkdir -p "${SERVER_DIR}/museum/config"
		cp /app/data/museum.yaml "${SERVER_DIR}/museum/config/museum.yaml"

		if [ -f "${SERVER_DIR}/museum/museum" ]; then
		echo "==> Found Museum server at ${SERVER_DIR}/museum"

		@@ -242,20 +295,25 @@ if [ -f "${SERVER_DIR}/museum/museum" ]; then
		if [ $i -eq 30 ]; then
		echo "==> ERROR: Museum server failed to start"
		echo "==> Please check logs at /app/data/logs/museum.log"
		exit 1
		tail -n 50 /app/data/logs/museum.log
		echo "==> Continuing anyway, but errors may occur later"
		fi
		done
		else
		echo "==> ERROR: Museum server not found at ${SERVER_DIR}/museum"
		echo "==> Please install the Museum server manually"
		exit 1
		echo "==> Continuing anyway, but errors may occur later"
		fi

		# Set up Caddy web server
		echo "==> Setting up Caddy web server"

		# Get local IP address
		LOCAL_IP=$(hostname -i 2>/dev/null \|\| echo "0.0.0.0")
		echo "==> Local IP address: $LOCAL_IP"

		# Create Caddy configuration file
		cat > /app/data/Caddyfile << 'EOF'
		cat > /app/data/Caddyfile << EOF
		{
		admin off
		}
		@@ -328,6 +386,9 @@ cat > /app/data/Caddyfile << 'EOF'
		}
		EOF

		echo "==> Caddy configuration created"
		cat /app/data/Caddyfile

		# Create runtime-config.js in writable location
		echo "==> Creating runtime-config.js in writable location"
		cat > /app/data/web/photos/static/runtime-config.js << 'EOF'
		@@ -406,309 +467,6 @@ cat > /app/data/web/photos/static/ente-patches.js << 'ENDPATCHES'
		}
		};

		// Comprehensive Buffer polyfill for SRP
		const originalBuffer = window.Buffer;
		window.Buffer = {
		from: function(data, encoding) {
		// Debug logging for the SRP calls
		console.debug('Buffer.from called with:',
		typeof data,
		data === undefined ? 'undefined' :
		data === null ? 'null' :
		Array.isArray(data) ? 'array[' + data.length + ']' :
		'value',
		'encoding:', encoding);

		// Handle undefined/null data - critical fix
		if (data === undefined \|\| data === null) {
		console.warn('Buffer.from called with ' + (data === undefined ? 'undefined' : 'null') + ' data, creating empty buffer');
		const result = {
		data: new Uint8Array(0),
		length: 0,
		toString: function(enc) { return ''; }
		};

		// Add additional methods that SRP might use
		result.slice = function() { return Buffer.from([]); };
		result.readUInt32BE = function() { return 0; };
		result.writeUInt32BE = function() { return result; };

		return result;
		}

		// Special case for hex strings - very important for SRP
		if (typeof data === 'string' && encoding === 'hex') {
		// Convert hex string to byte array
		const bytes = [];
		for (let i = 0; i < data.length; i += 2) {
		if (data.length - i >= 2) {
		bytes.push(parseInt(data.substr(i, 2), 16));
		}
		}

		const result = {
		data: new Uint8Array(bytes),
		length: bytes.length,
		toString: function(enc) {
		if (enc === 'hex' \|\| !enc) {
		return data; // Return original hex string
		}
		return bytes.map(b => String.fromCharCode(b)).join('');
		}
		};

		// Add methods needed by SRP
		result.slice = function(start, end) {
		const slicedData = bytes.slice(start, end);
		return Buffer.from(slicedData.map(b => b.toString(16).padStart(2, '0')).join(''), 'hex');
		};

		result.readUInt32BE = function(offset = 0) {
		let value = 0;
		for (let i = 0; i < 4; i++) {
		value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0);
		}
		return value;
		};

		result.writeUInt32BE = function(value, offset = 0) {
		for (let i = 0; i < 4; i++) {
		if (offset + i < bytes.length) {
		bytes[offset + 3 - i] = value & 0xFF;
		value >>>= 8;
		}
		}
		return result;
		};

		return result;
		}

		// Handle string data
		if (typeof data === 'string') {
		const bytes = Array.from(data).map(c => c.charCodeAt(0));
		const result = {
		data: new Uint8Array(bytes),
		length: bytes.length,
		toString: function(enc) {
		if (enc === 'hex') {
		return bytes.map(b => b.toString(16).padStart(2, '0')).join('');
		}
		return data;
		}
		};

		// Add SRP methods
		result.slice = function(start, end) {
		return Buffer.from(data.slice(start, end));
		};

		result.readUInt32BE = function(offset = 0) {
		let value = 0;
		for (let i = 0; i < 4; i++) {
		value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0);
		}
		return value;
		};

		result.writeUInt32BE = function(value, offset = 0) {
		for (let i = 0; i < 4; i++) {
		if (offset + i < bytes.length) {
		bytes[offset + 3 - i] = value & 0xFF;
		value >>>= 8;
		}
		}
		return result;
		};

		return result;
		}

		// Handle array/buffer data
		if (Array.isArray(data) \|\| ArrayBuffer.isView(data) \|\| (data instanceof ArrayBuffer)) {
		const bytes = Array.isArray(data) ? data : new Uint8Array(data.buffer \|\| data);
		const result = {
		data: new Uint8Array(bytes),
		length: bytes.length,
		toString: function(enc) {
		if (enc === 'hex') {
		return Array.from(bytes).map(b => b.toString(16).padStart(2, '0')).join('');
		}
		return Array.from(bytes).map(b => String.fromCharCode(b)).join('');
		}
		};

		// Add SRP methods
		result.slice = function(start, end) {
		return Buffer.from(bytes.slice(start, end));
		};

		result.readUInt32BE = function(offset = 0) {
		let value = 0;
		for (let i = 0; i < 4; i++) {
		value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0);
		}
		return value;
		};

		result.writeUInt32BE = function(value, offset = 0) {
		for (let i = 0; i < 4; i++) {
		if (offset + i < bytes.length) {
		bytes[offset + 3 - i] = value & 0xFF;
		value >>>= 8;
		}
		}
		return result;
		};

		return result;
		}

		// Handle object data (last resort)
		if (typeof data === 'object') {
		console.warn('Buffer.from called with object type', data);
		const result = {
		data: data,
		length: data.length \|\| 0,
		toString: function() { return JSON.stringify(data); }
		};

		// Add SRP methods
		result.slice = function() { return Buffer.from({}); };
		result.readUInt32BE = function() { return 0; };
		result.writeUInt32BE = function() { return result; };

		return result;
		}

		// Default fallback for any other type
		console.warn('Buffer.from called with unsupported type:', typeof data);
		const result = {
		data: new Uint8Array(0),
		length: 0,
		toString: function() { return ''; },
		slice: function() { return Buffer.from([]); },
		readUInt32BE: function() { return 0; },
		writeUInt32BE: function() { return result; }
		};

		return result;
		},

		isBuffer: function(obj) {
		return obj && (obj.data !== undefined \|\| (originalBuffer && originalBuffer.isBuffer && originalBuffer.isBuffer(obj)));
		},

		alloc: function(size, fill = 0) {
		const bytes = new Array(size).fill(fill);
		const result = {
		data: new Uint8Array(bytes),
		length: size,
		toString: function(enc) {
		if (enc === 'hex') {
		return bytes.map(b => b.toString(16).padStart(2, '0')).join('');
		}
		return bytes.map(b => String.fromCharCode(b)).join('');
		}
		};

		// Add SRP methods
		result.slice = function(start, end) {
		return Buffer.from(bytes.slice(start, end));
		};

		result.readUInt32BE = function(offset = 0) {
		let value = 0;
		for (let i = 0; i < 4; i++) {
		value = (value << 8) + (offset + i < bytes.length ? bytes[offset + i] : 0);
		}
		return value;
		};

		result.writeUInt32BE = function(value, offset = 0) {
		for (let i = 0; i < 4; i++) {
		if (offset + i < bytes.length) {
		bytes[offset + 3 - i] = value & 0xFF;
		value >>>= 8;
		}
		}
		return result;
		};

		return result;
		},

		concat: function(list) {
		if (!Array.isArray(list) \|\| list.length === 0) {
		return Buffer.alloc(0);
		}

		// Combine all buffers into one
		const totalLength = list.reduce((acc, buf) => acc + (buf ? (buf.length \|\| 0) : 0), 0);
		const combinedArray = new Uint8Array(totalLength);

		let offset = 0;
		for (const buf of list) {
		if (buf && buf.data) {
		const data = buf.data instanceof Uint8Array ? buf.data : new Uint8Array(buf.data);
		combinedArray.set(data, offset);
		offset += buf.length;
		}
		}

		const result = {
		data: combinedArray,
		length: totalLength,
		toString: function(enc) {
		if (enc === 'hex') {
		return Array.from(combinedArray).map(b => b.toString(16).padStart(2, '0')).join('');
		}
		return Array.from(combinedArray).map(b => String.fromCharCode(b)).join('');
		}
		};

		// Add SRP methods
		result.slice = function(start, end) {
		const slicedData = combinedArray.slice(start, end);
		return Buffer.from(slicedData);
		};

		result.readUInt32BE = function(offset = 0) {
		let value = 0;
		for (let i = 0; i < 4; i++) {
		value = (value << 8) + (offset + i < combinedArray.length ? combinedArray[offset + i] : 0);
		}
		return value;
		};

		result.writeUInt32BE = function(value, offset = 0) {
		for (let i = 0; i < 4; i++) {
		if (offset + i < combinedArray.length) {
		combinedArray[offset + 3 - i] = value & 0xFF;
		value >>>= 8;
		}
		}
		return result;
		};

		return result;
		}
		};

		// Patch the SRP implementation for browser compatibility
		if (!window.process) {
		window.process = {
		env: {
		NODE_ENV: 'production'
		}
		};
		}

		// Add any missing process methods
		window.process.nextTick = window.process.nextTick \|\| function(fn) {
		setTimeout(fn, 0);
		};

		console.log('Ente URL and SRP patches applied successfully');
		})();
		ENDPATCHES
		@@ -806,16 +564,142 @@ for app in photos accounts auth cast; do
		EOF
		else
		echo "==> Using existing $app web app"
		cp -r "/app/data/ente/web/$app/*" "/app/data/web/$app/"
		cp -r "/app/data/ente/web/$app/"* "/app/data/web/$app/"
		fi
		done

		# Install unzip if needed
		if ! command -v unzip &> /dev/null; then
		echo "==> Installing unzip (required for downloading repositories)"
		apt-get update
		apt-get install -y unzip
		fi

		# Start Caddy
		echo "==> Starting Caddy server"
		caddy run --config /app/data/Caddyfile --adapter caddyfile &

		# Kill any existing Caddy process to avoid port conflicts
		pkill -f "caddy run --config /app/data/Caddyfile" \|\| true
		sleep 1

		# Start Caddy with proper configuration
		caddy run --config /app/data/Caddyfile --adapter caddyfile > /app/data/logs/caddy.log 2>&1 &
		CADDY_PID=$!
		echo "==> Caddy server started with PID: $CADDY_PID"

		# Verify Caddy is listening
		sleep 5
		echo "==> Checking Caddy status:"
		if pgrep -f "caddy run --config /app/data/Caddyfile" > /dev/null; then
		echo "==> Caddy is running"
		else
		echo "==> WARNING: Caddy does not appear to be running"
		fi

		echo "==> Checking port 3080:"
		if netstat -tln \| grep ':3080' > /dev/null; then
		echo "==> Port 3080 is open and listening"
		else
		echo "==> WARNING: Port 3080 is not listening"
		netstat -tln

		# Try to start Caddy with explicit listen address
		echo "==> Trying to start Caddy with explicit listen address"
		cat > /app/data/Caddyfile << EOF
		{
		admin off
		}

		0.0.0.0:3080 {
		# API endpoints - proxy to Museum server
		handle /api/* {
		uri strip_prefix /api
		reverse_proxy localhost:8080
		}

		# Web applications static content
		handle /photos/* {
		uri strip_prefix /photos
		root * /app/data/web/photos
		try_files {path} {path}/ /index.html
		file_server
		}

		handle /accounts/* {
		uri strip_prefix /accounts
		root * /app/data/web/accounts
		try_files {path} {path}/ /index.html
		file_server
		}

		handle /auth/* {
		uri strip_prefix /auth
		root * /app/data/web/auth
		try_files {path} {path}/ /index.html
		file_server
		}

		handle /cast/* {
		uri strip_prefix /cast
		root * /app/data/web/cast
		try_files {path} {path}/ /index.html
		file_server
		}

		# Public albums handler
		handle /public/* {
		uri strip_prefix /public
		reverse_proxy localhost:8080/public
		}

		# Redirect root to photos
		handle / {
		redir /photos permanent
		}

		# Serve static files from photos by default
		handle {
		root * /app/data/web/photos
		try_files {path} {path}/ /index.html
		file_server
		}

		# Error handling
		handle_errors {
		respond "{http.error.status_code} {http.error.status_text}"
		}

		# Logging
		log {
		output file /app/data/logs/access.log
		format console
		level info
		}
		}
		EOF

		# Kill any existing Caddy process
		pkill -f "caddy run --config /app/data/Caddyfile" \|\| true
		sleep 1

		# Start Caddy again with explicit configuration
		caddy run --config /app/data/Caddyfile --adapter caddyfile > /app/data/logs/caddy.log 2>&1 &
		CADDY_PID=$!
		echo "==> Restarted Caddy server with PID: $CADDY_PID"
		sleep 5

		# Check again
		if netstat -tln \| grep ':3080' > /dev/null; then
		echo "==> Port 3080 is now open and listening"
		else
		echo "==> WARNING: Port 3080 is still not listening"
		netstat -tln
		fi
		fi

		# Check basic connectivity
		curl -v http://localhost:3080/ \|\| echo "==> WARNING: Cannot connect to localhost:3080"

		# Enter a wait state to catch signals
		echo "==> Entering wait state - watching logs for registration codes"
		echo "==> Registration verification codes will appear in the logs below"