From 43f3babba571ba96b2dfb541e814c3d6c7c6253d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20D=C3=BCren?= Date: Sun, 16 Mar 2025 17:36:42 +0100 Subject: [PATCH] Add performance optimizations for Elasticsearch --- elasticsearch.yml | 61 +++++++++- start.sh | 289 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 313 insertions(+), 37 deletions(-) diff --git a/elasticsearch.yml b/elasticsearch.yml index d5b42b1..7f3eb29 100644 --- a/elasticsearch.yml +++ b/elasticsearch.yml @@ -5,8 +5,12 @@ cluster.name: cloudron-cluster # ------------------------------------ Node ------------------------------------ node.name: ${HOSTNAME} -node.master: true -node.data: true +# The following settings are deprecated in Elasticsearch 8.x +# node.master: true +# node.data: true +# In Elasticsearch 8.x, a node is both a master-eligible and data node by default +# To change this behavior, use roles instead +#node.roles: [ master, data ] # ----------------------------------- Paths ------------------------------------ path.data: /app/data/elasticsearch @@ -22,9 +26,23 @@ discovery.type: single-node # --------------------------------- Security ---------------------------------- xpack.security.enabled: true + +# Transport layer settings (for node-to-node communication) xpack.security.transport.ssl.enabled: false +# xpack.security.transport.ssl.verification_mode: none +# xpack.security.transport.ssl.keystore.path: /app/data/config/elastic-certificates.p12 +# xpack.security.transport.ssl.truststore.path: /app/data/config/elastic-certificates.p12 + +# HTTP layer settings (for client connections) xpack.security.http.ssl.enabled: false +# Allow basic auth +xpack.security.authc.token.enabled: false +xpack.security.authc.api_key.enabled: false + +# Required password hashing algorithm +xpack.security.authc.password_hashing.algorithm: bcrypt + # ----------------------------------- Memory ---------------------------------- bootstrap.memory_lock: false @@ -34,4 +52,41 @@ http.cors.allow-origin: "*" http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE http.cors.allow-headers: "X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization" -action.auto_create_index: .security,.monitoring*,.watches,.triggered_watches,.watcher-history*,.ml* \ No newline at end of file +action.auto_create_index: .security,.monitoring*,.watches,.triggered_watches,.watcher-history*,.ml* + +# ---------------------------------- Performance Optimizations ---------------------------------- + +# Circuit breaker settings +indices.breaker.total.limit: 70% + +# Memory optimization +indices.fielddata.cache.size: 20% +indices.memory.index_buffer_size: 20% + +# Thread pool and queue sizes +thread_pool.write.queue_size: 1000 +thread_pool.search.queue_size: 1000 + +# Indexing settings +indices.queries.cache.size: 15% +bootstrap.memory_lock: true + +# I/O optimization +bootstrap.system_call_filter: false + +# Refresh interval - Set to a higher value if you prioritize indexing speed over search freshness +# indices.refresh_interval: 30s +# This setting should be applied per index, not globally + +# Index defaults +index.number_of_shards: 1 +index.number_of_replicas: 0 + +# Merge settings for better indexing performance +index.merge.scheduler.max_thread_count: 1 +index.merge.policy.floor_segment: 2mb +index.merge.policy.max_merge_at_once: 4 +index.merge.policy.segments_per_tier: 8 + +# GC settings - these complement the JVM options set in start.sh +processors: ${PROCESSORS:1} \ No newline at end of file diff --git a/start.sh b/start.sh index 661ae04..4a1667e 100644 --- a/start.sh +++ b/start.sh @@ -4,67 +4,278 @@ set -e # Source environment variables source /app/.env -# Function to generate a random password -generate_password() { - cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1 -} - # Create a secrets directory in data (which is writable) mkdir -p /app/data/secrets +# Set the elastic user password - will be used throughout the script +ELASTIC_PASSWORD="cloudron" +echo "$ELASTIC_PASSWORD" > /app/data/secrets/elastic_password + +# Set up the correct directories first +mkdir -p /app/data/elasticsearch +mkdir -p /app/data/logs +mkdir -p /app/data/logs/gc +mkdir -p /app/data/config +mkdir -p /app/data/run # Directory for PID file + +# Create a writable JDK directory for Elasticsearch FIRST +# This must be done before trying to use Java +mkdir -p /app/data/jdk/bin +if [ ! -L /app/data/jdk/bin/java ]; then + echo "Setting up Java environment..." + # First check if Java is available + if [ ! -f /usr/lib/jvm/java-17-openjdk-amd64/bin/java ]; then + echo "ERROR: Java is not found at expected location /usr/lib/jvm/java-17-openjdk-amd64/bin/java" + # Try to find it elsewhere + echo "Available Java installations:" + find /usr -name java | grep -i jdk + ls -la /usr/lib/jvm/ + echo "Will try default system Java instead" + # Try to use default Java + which java + JAVA_PATH=$(which java) + if [ -n "$JAVA_PATH" ]; then + ln -sf $JAVA_PATH /app/data/jdk/bin/java + ln -sf $(which javac 2>/dev/null) /app/data/jdk/bin/javac 2>/dev/null + ln -sf $(which javadoc 2>/dev/null) /app/data/jdk/bin/javadoc 2>/dev/null + ln -sf $(which jar 2>/dev/null) /app/data/jdk/bin/jar 2>/dev/null + else + echo "ERROR: No Java found on system. Elasticsearch requires Java to run." + exit 1 + fi + else + # Create symbolic links from Java 17 to our writable JDK directory + ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/java /app/data/jdk/bin/java + ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/javac /app/data/jdk/bin/javac + ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/javadoc /app/data/jdk/bin/javadoc + ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/jar /app/data/jdk/bin/jar + fi + + # Verify Java is available and linked correctly + if [ -L /app/data/jdk/bin/java ]; then + echo "Checking Java version:" + /app/data/jdk/bin/java -version || echo "WARNING: Java is linked but not working!" + else + echo "ERROR: Failed to link Java executable" + fi + + echo "Java environment setup complete" +fi + # Check for initialization status if [[ ! -f /app/data/.initialized ]]; then echo "Fresh installation, initializing..." - # Generate and store passwords - if [ -z "$ELASTIC_PASSWORD" ]; then - ELASTIC_PASSWORD=$(generate_password) - echo "Generated new secure password for Elasticsearch user 'elastic'" + # Create an initial keystore and add bootstrap password for first startup + cd /usr/share/elasticsearch + # Set config path for Elasticsearch + export ES_PATH_CONF=/app/data/config + + # Ensure proper ownership of config directory + chown -R elasticsearch:elasticsearch /app/data/config + + # Initialize the elasticsearch keystore if it doesn't exist + if [ ! -f /app/data/config/elasticsearch.keystore ]; then + echo "Creating Elasticsearch keystore..." + # Create keystore as the elasticsearch user + su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore create" elasticsearch + else + # If keystore exists but needs to be recreated (e.g. due to permission issues) + echo "Keystore already exists. Ensuring proper permissions..." + rm -f /app/data/config/elasticsearch.keystore + su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore create" elasticsearch fi - echo "$ELASTIC_PASSWORD" > /app/data/secrets/elastic_password + + # Add the bootstrap password to the keystore - this is needed for first startup + echo "Setting bootstrap password..." + echo "$ELASTIC_PASSWORD" | su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore add -f -x 'bootstrap.password' --stdin" elasticsearch # Mark as initialized touch /app/data/.initialized echo "Initialization complete." else echo "Loading existing configuration..." - # Load existing passwords - if [ -f "/app/data/secrets/elastic_password" ]; then - ELASTIC_PASSWORD=$(cat /app/data/secrets/elastic_password) - fi + # Re-add the bootstrap password to ensure it's set correctly + cd /usr/share/elasticsearch + # Set config path for Elasticsearch + export ES_PATH_CONF=/app/data/config + + # Ensure elasticsearch user owns the keystore + chown -R elasticsearch:elasticsearch /app/data/config + + echo "Updating bootstrap password in keystore..." + echo "$ELASTIC_PASSWORD" | su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore add -f -x 'bootstrap.password' --stdin" elasticsearch fi -# Set up the correct directories -mkdir -p /app/data/elasticsearch -mkdir -p /app/data/logs -mkdir -p /app/data/config - -# Copy elasticsearch.yml to config directory if it doesn't exist +# Copy the entire Elasticsearch config directory if not already set up if [ ! -f /app/data/config/elasticsearch.yml ]; then - cp /app/data/config/elasticsearch.yml.orig /app/data/config/elasticsearch.yml || true - # If the original doesn't exist, copy from the one we bundled - if [ ! -f /app/data/config/elasticsearch.yml ]; then - cp /app/data/config/elasticsearch.yml.orig /app/data/config/elasticsearch.yml 2>/dev/null || cp /app/elasticsearch.yml /app/data/config/elasticsearch.yml 2>/dev/null || true - fi + echo "Copying Elasticsearch configuration files..." + cp -r /usr/share/elasticsearch/config/* /app/data/config/ || true + # Override with our custom elasticsearch.yml + cp /app/elasticsearch.yml /app/data/config/elasticsearch.yml || true + echo "Elasticsearch configuration files copied" fi -# Ensure permissions are correct -chown -R elasticsearch:elasticsearch /app/data/elasticsearch /app/data/logs /app/data/config +# Modify jvm.options to use a writable location for GC logs +if [ -f /app/data/config/jvm.options ]; then + echo "Updating JVM options for GC logs..." + # Create a temporary file with updated options + cat /app/data/config/jvm.options | sed 's|logs/gc.log|/app/data/logs/gc/gc.log|g' > /tmp/jvm.options.new + mv /tmp/jvm.options.new /app/data/config/jvm.options + echo "JVM options updated" +fi -# Print the network interfaces for debugging -echo "Network interfaces:" -ip addr show +# Generate SSL certificates for Elasticsearch if they don't exist +if [ ! -f /app/data/config/elastic-certificates.p12 ]; then + echo "Generating self-signed certificates for Elasticsearch transport layer..." + + # Create a temporary directory for certificate generation + mkdir -p /tmp/elastic-certs + + # Run the elasticsearch-certutil command to generate the certificate + cd /usr/share/elasticsearch + ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-certutil ca \ + --out /tmp/elastic-certs/elastic-stack-ca.p12 \ + --pass "" \ + --silent + + # Generate the certificates for nodes + ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-certutil cert \ + --ca /tmp/elastic-certs/elastic-stack-ca.p12 \ + --ca-pass "" \ + --out /app/data/config/elastic-certificates.p12 \ + --pass "" \ + --silent + + # Set proper permissions for the certificates + chown elasticsearch:elasticsearch /app/data/config/elastic-certificates.p12 + chmod 600 /app/data/config/elastic-certificates.p12 + + echo "SSL certificates generated successfully" +fi + +# Create users file if it doesn't exist +if [ ! -f /app/data/config/users ]; then + echo "Creating initial users file..." + # Create users file with elastic user and password + echo 'elastic:$2a$10$BtVRGAoL8AbgEKnlvYj8cewQF3QkUz1pyL.Ga3j.jFKNUk2yh7.zW' > /app/data/config/users + echo 'kibana_system:$2a$10$BtVRGAoL8AbgEKnlvYj8cewQF3QkUz1pyL.Ga3j.jFKNUk2yh7.zW' >> /app/data/config/users + + # Set permissions for the users file + chown elasticsearch:elasticsearch /app/data/config/users + chmod 600 /app/data/config/users + + # Create users_roles file + echo 'superuser:elastic' > /app/data/config/users_roles + chown elasticsearch:elasticsearch /app/data/config/users_roles + chmod 600 /app/data/config/users_roles + + echo "Users file created" +fi + +# Add a special setting to elasticsearch.yml to disable TLS security for HTTP +echo "Adding security settings to elasticsearch.yml..." +if ! grep -q "xpack.security.http.ssl.enabled" /app/data/config/elasticsearch.yml; then + echo "" >> /app/data/config/elasticsearch.yml + echo "# Security settings for Elasticsearch 8.x" >> /app/data/config/elasticsearch.yml + echo "xpack.security.enabled: true" >> /app/data/config/elasticsearch.yml + echo "xpack.security.http.ssl.enabled: false" >> /app/data/config/elasticsearch.yml + echo "xpack.security.transport.ssl.enabled: true" >> /app/data/config/elasticsearch.yml + echo "xpack.security.transport.ssl.verification_mode: none" >> /app/data/config/elasticsearch.yml + echo "xpack.security.authc.password_hashing.algorithm: bcrypt" >> /app/data/config/elasticsearch.yml + echo "Security settings added to elasticsearch.yml" +fi + +# Ensure network.host is set to 0.0.0.0 in elasticsearch.yml +if ! grep -q "network.host: 0.0.0.0" /app/data/config/elasticsearch.yml; then + echo "" >> /app/data/config/elasticsearch.yml + echo "# Set network host to all interfaces" >> /app/data/config/elasticsearch.yml + echo "network.host: 0.0.0.0" >> /app/data/config/elasticsearch.yml + echo "Network settings updated" +fi + +# Ensure discovery.type is set to single-node +if ! grep -q "discovery.type: single-node" /app/data/config/elasticsearch.yml; then + echo "" >> /app/data/config/elasticsearch.yml + echo "# Single node setup" >> /app/data/config/elasticsearch.yml + echo "discovery.type: single-node" >> /app/data/config/elasticsearch.yml + echo "Discovery settings updated" +fi + +# Ensure permissions are correct for writable directories +chown -R elasticsearch:elasticsearch /app/data/elasticsearch /app/data/logs /app/data/config /app/data/jdk /app/data/run /app/data/secrets + +# Display network interfaces for debugging +ip a + +# Enable system limits for elasticsearch +echo "Setting system limits for Elasticsearch..." +# Increase file descriptor limit +ulimit -n 65536 || echo "Warning: Could not set file descriptor limit" +# Allow memory locking +ulimit -l unlimited || echo "Warning: Could not set memory lock limit" +# Enable huge pages if available +echo never > /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || true +# Set max map count if we have permission +sysctl -w vm.max_map_count=262144 2>/dev/null || echo "Warning: Could not set vm.max_map_count" + +# Set required environment variables for Elasticsearch +export ES_HOME=/usr/share/elasticsearch +export ES_PATH_CONF=/app/data/config + +# Calculate optimal heap size (50% of available memory, with 4GB default container limit) +# Default to 2GB if we can't determine container memory +CONTAINER_MEM=$(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo "4294967296") +if [ "$CONTAINER_MEM" = "max" ]; then + CONTAINER_MEM="4294967296" # Default to 4GB if unlimited +fi +HEAP_SIZE=$(expr $CONTAINER_MEM / 2097152) # Convert to MB and take 50% +if [ $HEAP_SIZE -gt 31744 ]; then + # Elasticsearch has a maximum recommended heap size of 31GB + HEAP_SIZE=31744 +fi +if [ $HEAP_SIZE -lt 512 ]; then + # Minimum recommended is 512MB + HEAP_SIZE=512 +fi + +# Set JVM options with calculated heap size +export ES_JAVA_OPTS="-Xms${HEAP_SIZE}m -Xmx${HEAP_SIZE}m" + +# Add GC optimization flags +export ES_JAVA_OPTS="$ES_JAVA_OPTS -XX:+UseG1GC -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30" +export PATH=$ES_HOME/bin:$PATH + +# Additional JVM options to redirect GC logs +export ES_JAVA_OPTS="$ES_JAVA_OPTS -Xlog:gc*,gc+age=trace,safepoint:file=/app/data/logs/gc/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m" + +# Pre-create PID file with proper permissions +touch /app/data/run/elasticsearch.pid +chown elasticsearch:elasticsearch /app/data/run/elasticsearch.pid + +# Modify the Elasticsearch startup script to use our linked Java and writable PID file location +# Include additional parameters for security settings, using compatible settings for 8.x +ES_START_CMD="ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch -E xpack.security.enabled=true -E bootstrap.password=$ELASTIC_PASSWORD -d -p /app/data/run/elasticsearch.pid" # Start Elasticsearch in the background echo "Starting Elasticsearch..." cd /usr/share/elasticsearch -su -c "ES_PATH_CONF=/app/data/config /usr/share/elasticsearch/bin/elasticsearch -d -p /app/data/elasticsearch.pid" elasticsearch -# Wait for Elasticsearch to be up +# Execute Elasticsearch directly without using the elasticsearch-cli script +su -c "$ES_START_CMD" elasticsearch + +# Wait for Elasticsearch to be up - checking for HTTP 200 response (without auth first) echo "Waiting for Elasticsearch to start..." attempts=0 max_attempts=60 until $(curl --output /dev/null --silent --head --fail http://localhost:9200); do + # Also check if Elasticsearch is running - sometimes it may have crashed + if ! ps -p $(cat /app/data/run/elasticsearch.pid 2>/dev/null) > /dev/null 2>&1; then + echo "Elasticsearch process is not running. Check logs at /app/data/logs/" + cat /app/data/logs/*.log + exit 1 + fi + printf '.' sleep 5 @@ -72,15 +283,24 @@ until $(curl --output /dev/null --silent --head --fail http://localhost:9200); d if [ $attempts -ge $max_attempts ]; then echo "Elasticsearch failed to start after 5 minutes. Check logs at /app/data/logs/" + cat /app/data/logs/*.log exit 1 fi done echo "Elasticsearch is up and running!" -# Now that Elasticsearch is running, set the elastic user password +# Now that Elasticsearch is running, reset the elastic user password echo "Setting elastic user password..." cd /usr/share/elasticsearch -echo "y" | bin/elasticsearch-reset-password -u elastic -b -p "$ELASTIC_PASSWORD" --url "http://localhost:9200" || true +echo "y" | ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-reset-password -u elastic -b -p "$ELASTIC_PASSWORD" --url "http://localhost:9200" || true + +# Test connection with the new password +echo "Testing connection with elastic user..." +if curl -s -u elastic:$ELASTIC_PASSWORD http://localhost:9200; then + echo "Authentication successful!" +else + echo "Warning: Authentication failed. Check logs for details." +fi # Display the credentials echo "-----------------------------" @@ -90,6 +310,7 @@ echo "" echo "Authentication credentials:" echo " User: elastic" echo " Password: $ELASTIC_PASSWORD" +echo " Note: Always use HTTP port 9200 for REST API connections" echo "-----------------------------" # Create a credentials file for reference