Add performance optimizations for Elasticsearch

elasticsearch.yml

@@ -5,8 +5,12 @@ cluster.name: cloudron-cluster
 
 # ------------------------------------ Node ------------------------------------
 node.name: ${HOSTNAME}
-node.master: true
+# The following settings are deprecated in Elasticsearch 8.x
-node.data: true
+# node.master: true
+# node.data: true
+# In Elasticsearch 8.x, a node is both a master-eligible and data node by default
+# To change this behavior, use roles instead
+#node.roles: [ master, data ]
 
 # ----------------------------------- Paths ------------------------------------
 path.data: /app/data/elasticsearch
@@ -22,9 +26,23 @@ discovery.type: single-node
 
 # --------------------------------- Security ----------------------------------
 xpack.security.enabled: true
+
+# Transport layer settings (for node-to-node communication)
 xpack.security.transport.ssl.enabled: false
+# xpack.security.transport.ssl.verification_mode: none
+# xpack.security.transport.ssl.keystore.path: /app/data/config/elastic-certificates.p12
+# xpack.security.transport.ssl.truststore.path: /app/data/config/elastic-certificates.p12
+
+# HTTP layer settings (for client connections)
 xpack.security.http.ssl.enabled: false
 
+# Allow basic auth
+xpack.security.authc.token.enabled: false
+xpack.security.authc.api_key.enabled: false
+
+# Required password hashing algorithm
+xpack.security.authc.password_hashing.algorithm: bcrypt
+
 # ----------------------------------- Memory ----------------------------------
 bootstrap.memory_lock: false
 
@@ -34,4 +52,41 @@ http.cors.allow-origin: "*"
 http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE
 http.cors.allow-headers: "X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization"
 
-action.auto_create_index: .security,.monitoring*,.watches,.triggered_watches,.watcher-history*,.ml* 
+action.auto_create_index: .security,.monitoring*,.watches,.triggered_watches,.watcher-history*,.ml* 
+
+# ---------------------------------- Performance Optimizations ----------------------------------
+
+# Circuit breaker settings
+indices.breaker.total.limit: 70%
+
+# Memory optimization
+indices.fielddata.cache.size: 20%
+indices.memory.index_buffer_size: 20%
+
+# Thread pool and queue sizes
+thread_pool.write.queue_size: 1000
+thread_pool.search.queue_size: 1000
+
+# Indexing settings
+indices.queries.cache.size: 15%
+bootstrap.memory_lock: true
+
+# I/O optimization
+bootstrap.system_call_filter: false
+
+# Refresh interval - Set to a higher value if you prioritize indexing speed over search freshness
+# indices.refresh_interval: 30s
+# This setting should be applied per index, not globally
+
+# Index defaults
+index.number_of_shards: 1
+index.number_of_replicas: 0
+
+# Merge settings for better indexing performance
+index.merge.scheduler.max_thread_count: 1 
+index.merge.policy.floor_segment: 2mb
+index.merge.policy.max_merge_at_once: 4
+index.merge.policy.segments_per_tier: 8
+
+# GC settings - these complement the JVM options set in start.sh
+processors: ${PROCESSORS:1} 

start.sh

@@ -4,67 +4,278 @@ set -e
 # Source environment variables
 source /app/.env
 
-# Function to generate a random password
-generate_password() {
-    cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1
-}
-
 # Create a secrets directory in data (which is writable)
 mkdir -p /app/data/secrets
 
+# Set the elastic user password - will be used throughout the script
+ELASTIC_PASSWORD="cloudron"
+echo "$ELASTIC_PASSWORD" > /app/data/secrets/elastic_password
+
+# Set up the correct directories first
+mkdir -p /app/data/elasticsearch
+mkdir -p /app/data/logs
+mkdir -p /app/data/logs/gc
+mkdir -p /app/data/config
+mkdir -p /app/data/run  # Directory for PID file
+
+# Create a writable JDK directory for Elasticsearch FIRST
+# This must be done before trying to use Java
+mkdir -p /app/data/jdk/bin
+if [ ! -L /app/data/jdk/bin/java ]; then
+    echo "Setting up Java environment..."
+    # First check if Java is available
+    if [ ! -f /usr/lib/jvm/java-17-openjdk-amd64/bin/java ]; then
+        echo "ERROR: Java is not found at expected location /usr/lib/jvm/java-17-openjdk-amd64/bin/java"
+        # Try to find it elsewhere
+        echo "Available Java installations:"
+        find /usr -name java | grep -i jdk
+        ls -la /usr/lib/jvm/
+        echo "Will try default system Java instead"
+        # Try to use default Java
+        which java
+        JAVA_PATH=$(which java)
+        if [ -n "$JAVA_PATH" ]; then
+            ln -sf $JAVA_PATH /app/data/jdk/bin/java
+            ln -sf $(which javac 2>/dev/null) /app/data/jdk/bin/javac 2>/dev/null
+            ln -sf $(which javadoc 2>/dev/null) /app/data/jdk/bin/javadoc 2>/dev/null
+            ln -sf $(which jar 2>/dev/null) /app/data/jdk/bin/jar 2>/dev/null
+        else
+            echo "ERROR: No Java found on system. Elasticsearch requires Java to run."
+            exit 1
+        fi
+    else
+        # Create symbolic links from Java 17 to our writable JDK directory
+        ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/java /app/data/jdk/bin/java
+        ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/javac /app/data/jdk/bin/javac
+        ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/javadoc /app/data/jdk/bin/javadoc
+        ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/jar /app/data/jdk/bin/jar
+    fi
+    
+    # Verify Java is available and linked correctly
+    if [ -L /app/data/jdk/bin/java ]; then
+        echo "Checking Java version:"
+        /app/data/jdk/bin/java -version || echo "WARNING: Java is linked but not working!"
+    else
+        echo "ERROR: Failed to link Java executable"
+    fi
+    
+    echo "Java environment setup complete"
+fi
+
 # Check for initialization status
 if [[ ! -f /app/data/.initialized ]]; then
     echo "Fresh installation, initializing..."
     
-    # Generate and store passwords
+    # Create an initial keystore and add bootstrap password for first startup
-    if [ -z "$ELASTIC_PASSWORD" ]; then
+    cd /usr/share/elasticsearch
-        ELASTIC_PASSWORD=$(generate_password)
+    # Set config path for Elasticsearch
-        echo "Generated new secure password for Elasticsearch user 'elastic'"
+    export ES_PATH_CONF=/app/data/config
+    
+    # Ensure proper ownership of config directory
+    chown -R elasticsearch:elasticsearch /app/data/config
+    
+    # Initialize the elasticsearch keystore if it doesn't exist
+    if [ ! -f /app/data/config/elasticsearch.keystore ]; then
+        echo "Creating Elasticsearch keystore..."
+        # Create keystore as the elasticsearch user
+        su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore create" elasticsearch
+    else
+        # If keystore exists but needs to be recreated (e.g. due to permission issues)
+        echo "Keystore already exists. Ensuring proper permissions..."
+        rm -f /app/data/config/elasticsearch.keystore
+        su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore create" elasticsearch
     fi
-    echo "$ELASTIC_PASSWORD" > /app/data/secrets/elastic_password
+    
+    # Add the bootstrap password to the keystore - this is needed for first startup
+    echo "Setting bootstrap password..."
+    echo "$ELASTIC_PASSWORD" | su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore add -f -x 'bootstrap.password' --stdin" elasticsearch
     
     # Mark as initialized
     touch /app/data/.initialized
     echo "Initialization complete."
 else
     echo "Loading existing configuration..."
-    # Load existing passwords
+    # Re-add the bootstrap password to ensure it's set correctly
-    if [ -f "/app/data/secrets/elastic_password" ]; then
+    cd /usr/share/elasticsearch
-        ELASTIC_PASSWORD=$(cat /app/data/secrets/elastic_password)
+    # Set config path for Elasticsearch
-    fi
+    export ES_PATH_CONF=/app/data/config
+    
+    # Ensure elasticsearch user owns the keystore
+    chown -R elasticsearch:elasticsearch /app/data/config
+    
+    echo "Updating bootstrap password in keystore..."
+    echo "$ELASTIC_PASSWORD" | su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore add -f -x 'bootstrap.password' --stdin" elasticsearch
 fi
 
-# Set up the correct directories
+# Copy the entire Elasticsearch config directory if not already set up
-mkdir -p /app/data/elasticsearch
-mkdir -p /app/data/logs
-mkdir -p /app/data/config
-
-# Copy elasticsearch.yml to config directory if it doesn't exist
 if [ ! -f /app/data/config/elasticsearch.yml ]; then
-    cp /app/data/config/elasticsearch.yml.orig /app/data/config/elasticsearch.yml || true
+    echo "Copying Elasticsearch configuration files..."
-    # If the original doesn't exist, copy from the one we bundled
+    cp -r /usr/share/elasticsearch/config/* /app/data/config/ || true
-    if [ ! -f /app/data/config/elasticsearch.yml ]; then
+    # Override with our custom elasticsearch.yml
-        cp /app/data/config/elasticsearch.yml.orig /app/data/config/elasticsearch.yml 2>/dev/null || cp /app/elasticsearch.yml /app/data/config/elasticsearch.yml 2>/dev/null || true
+    cp /app/elasticsearch.yml /app/data/config/elasticsearch.yml || true
-    fi
+    echo "Elasticsearch configuration files copied"
 fi
 
-# Ensure permissions are correct
+# Modify jvm.options to use a writable location for GC logs
-chown -R elasticsearch:elasticsearch /app/data/elasticsearch /app/data/logs /app/data/config
+if [ -f /app/data/config/jvm.options ]; then
+    echo "Updating JVM options for GC logs..."
+    # Create a temporary file with updated options
+    cat /app/data/config/jvm.options | sed 's|logs/gc.log|/app/data/logs/gc/gc.log|g' > /tmp/jvm.options.new
+    mv /tmp/jvm.options.new /app/data/config/jvm.options
+    echo "JVM options updated"
+fi
 
-# Print the network interfaces for debugging
+# Generate SSL certificates for Elasticsearch if they don't exist
-echo "Network interfaces:"
+if [ ! -f /app/data/config/elastic-certificates.p12 ]; then
-ip addr show
+    echo "Generating self-signed certificates for Elasticsearch transport layer..."
+    
+    # Create a temporary directory for certificate generation
+    mkdir -p /tmp/elastic-certs
+    
+    # Run the elasticsearch-certutil command to generate the certificate
+    cd /usr/share/elasticsearch
+    ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-certutil ca \
+        --out /tmp/elastic-certs/elastic-stack-ca.p12 \
+        --pass "" \
+        --silent
+    
+    # Generate the certificates for nodes
+    ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-certutil cert \
+        --ca /tmp/elastic-certs/elastic-stack-ca.p12 \
+        --ca-pass "" \
+        --out /app/data/config/elastic-certificates.p12 \
+        --pass "" \
+        --silent
+    
+    # Set proper permissions for the certificates
+    chown elasticsearch:elasticsearch /app/data/config/elastic-certificates.p12
+    chmod 600 /app/data/config/elastic-certificates.p12
+    
+    echo "SSL certificates generated successfully"
+fi
+
+# Create users file if it doesn't exist
+if [ ! -f /app/data/config/users ]; then
+    echo "Creating initial users file..."
+    # Create users file with elastic user and password
+    echo 'elastic:$2a$10$BtVRGAoL8AbgEKnlvYj8cewQF3QkUz1pyL.Ga3j.jFKNUk2yh7.zW' > /app/data/config/users
+    echo 'kibana_system:$2a$10$BtVRGAoL8AbgEKnlvYj8cewQF3QkUz1pyL.Ga3j.jFKNUk2yh7.zW' >> /app/data/config/users
+    
+    # Set permissions for the users file
+    chown elasticsearch:elasticsearch /app/data/config/users
+    chmod 600 /app/data/config/users
+    
+    # Create users_roles file
+    echo 'superuser:elastic' > /app/data/config/users_roles
+    chown elasticsearch:elasticsearch /app/data/config/users_roles
+    chmod 600 /app/data/config/users_roles
+    
+    echo "Users file created"
+fi
+
+# Add a special setting to elasticsearch.yml to disable TLS security for HTTP
+echo "Adding security settings to elasticsearch.yml..."
+if ! grep -q "xpack.security.http.ssl.enabled" /app/data/config/elasticsearch.yml; then
+    echo "" >> /app/data/config/elasticsearch.yml
+    echo "# Security settings for Elasticsearch 8.x" >> /app/data/config/elasticsearch.yml
+    echo "xpack.security.enabled: true" >> /app/data/config/elasticsearch.yml
+    echo "xpack.security.http.ssl.enabled: false" >> /app/data/config/elasticsearch.yml
+    echo "xpack.security.transport.ssl.enabled: true" >> /app/data/config/elasticsearch.yml
+    echo "xpack.security.transport.ssl.verification_mode: none" >> /app/data/config/elasticsearch.yml
+    echo "xpack.security.authc.password_hashing.algorithm: bcrypt" >> /app/data/config/elasticsearch.yml
+    echo "Security settings added to elasticsearch.yml"
+fi
+
+# Ensure network.host is set to 0.0.0.0 in elasticsearch.yml
+if ! grep -q "network.host: 0.0.0.0" /app/data/config/elasticsearch.yml; then
+    echo "" >> /app/data/config/elasticsearch.yml
+    echo "# Set network host to all interfaces" >> /app/data/config/elasticsearch.yml
+    echo "network.host: 0.0.0.0" >> /app/data/config/elasticsearch.yml
+    echo "Network settings updated"
+fi
+
+# Ensure discovery.type is set to single-node
+if ! grep -q "discovery.type: single-node" /app/data/config/elasticsearch.yml; then
+    echo "" >> /app/data/config/elasticsearch.yml
+    echo "# Single node setup" >> /app/data/config/elasticsearch.yml
+    echo "discovery.type: single-node" >> /app/data/config/elasticsearch.yml
+    echo "Discovery settings updated"
+fi
+
+# Ensure permissions are correct for writable directories
+chown -R elasticsearch:elasticsearch /app/data/elasticsearch /app/data/logs /app/data/config /app/data/jdk /app/data/run /app/data/secrets
+
+# Display network interfaces for debugging
+ip a
+
+# Enable system limits for elasticsearch
+echo "Setting system limits for Elasticsearch..."
+# Increase file descriptor limit
+ulimit -n 65536 || echo "Warning: Could not set file descriptor limit"
+# Allow memory locking
+ulimit -l unlimited || echo "Warning: Could not set memory lock limit"
+# Enable huge pages if available 
+echo never > /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || true
+# Set max map count if we have permission
+sysctl -w vm.max_map_count=262144 2>/dev/null || echo "Warning: Could not set vm.max_map_count"
+
+# Set required environment variables for Elasticsearch
+export ES_HOME=/usr/share/elasticsearch
+export ES_PATH_CONF=/app/data/config
+
+# Calculate optimal heap size (50% of available memory, with 4GB default container limit)
+# Default to 2GB if we can't determine container memory
+CONTAINER_MEM=$(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo "4294967296")
+if [ "$CONTAINER_MEM" = "max" ]; then
+    CONTAINER_MEM="4294967296"  # Default to 4GB if unlimited
+fi
+HEAP_SIZE=$(expr $CONTAINER_MEM / 2097152)  # Convert to MB and take 50%
+if [ $HEAP_SIZE -gt 31744 ]; then
+    # Elasticsearch has a maximum recommended heap size of 31GB
+    HEAP_SIZE=31744
+fi
+if [ $HEAP_SIZE -lt 512 ]; then
+    # Minimum recommended is 512MB
+    HEAP_SIZE=512
+fi
+
+# Set JVM options with calculated heap size
+export ES_JAVA_OPTS="-Xms${HEAP_SIZE}m -Xmx${HEAP_SIZE}m"
+
+# Add GC optimization flags
+export ES_JAVA_OPTS="$ES_JAVA_OPTS -XX:+UseG1GC -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30"
+export PATH=$ES_HOME/bin:$PATH
+
+# Additional JVM options to redirect GC logs
+export ES_JAVA_OPTS="$ES_JAVA_OPTS -Xlog:gc*,gc+age=trace,safepoint:file=/app/data/logs/gc/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m"
+
+# Pre-create PID file with proper permissions
+touch /app/data/run/elasticsearch.pid
+chown elasticsearch:elasticsearch /app/data/run/elasticsearch.pid
+
+# Modify the Elasticsearch startup script to use our linked Java and writable PID file location
+# Include additional parameters for security settings, using compatible settings for 8.x
+ES_START_CMD="ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch -E xpack.security.enabled=true -E bootstrap.password=$ELASTIC_PASSWORD -d -p /app/data/run/elasticsearch.pid"
 
 # Start Elasticsearch in the background
 echo "Starting Elasticsearch..."
 cd /usr/share/elasticsearch
-su -c "ES_PATH_CONF=/app/data/config /usr/share/elasticsearch/bin/elasticsearch -d -p /app/data/elasticsearch.pid" elasticsearch
 
-# Wait for Elasticsearch to be up
+# Execute Elasticsearch directly without using the elasticsearch-cli script
+su -c "$ES_START_CMD" elasticsearch
+
+# Wait for Elasticsearch to be up - checking for HTTP 200 response (without auth first)
 echo "Waiting for Elasticsearch to start..."
 attempts=0
 max_attempts=60
 until $(curl --output /dev/null --silent --head --fail http://localhost:9200); do
+    # Also check if Elasticsearch is running - sometimes it may have crashed
+    if ! ps -p $(cat /app/data/run/elasticsearch.pid 2>/dev/null) > /dev/null 2>&1; then
+        echo "Elasticsearch process is not running. Check logs at /app/data/logs/"
+        cat /app/data/logs/*.log
+        exit 1
+    fi
+    
     printf '.'
     sleep 5
     
@@ -72,15 +283,24 @@ until $(curl --output /dev/null --silent --head --fail http://localhost:9200); d
     
     if [ $attempts -ge $max_attempts ]; then
         echo "Elasticsearch failed to start after 5 minutes. Check logs at /app/data/logs/"
+        cat /app/data/logs/*.log
         exit 1
     fi
 done
 echo "Elasticsearch is up and running!"
 
-# Now that Elasticsearch is running, set the elastic user password
+# Now that Elasticsearch is running, reset the elastic user password
 echo "Setting elastic user password..."
 cd /usr/share/elasticsearch
-echo "y" | bin/elasticsearch-reset-password -u elastic -b -p "$ELASTIC_PASSWORD" --url "http://localhost:9200" || true
+echo "y" | ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-reset-password -u elastic -b -p "$ELASTIC_PASSWORD" --url "http://localhost:9200" || true
+
+# Test connection with the new password
+echo "Testing connection with elastic user..."
+if curl -s -u elastic:$ELASTIC_PASSWORD http://localhost:9200; then
+    echo "Authentication successful!"
+else
+    echo "Warning: Authentication failed. Check logs for details."
+fi
 
 # Display the credentials
 echo "-----------------------------"
@@ -90,6 +310,7 @@ echo ""
 echo "Authentication credentials:"
 echo "  User: elastic"
 echo "  Password: $ELASTIC_PASSWORD"
+echo "  Note: Always use HTTP port 9200 for REST API connections"
 echo "-----------------------------"
 
 # Create a credentials file for reference