Add performance optimizations for Elasticsearch

# ------------------------------------ Node ------------------------------------

node.name: ${HOSTNAME}

node.master: true

node.data: true

# The following settings are deprecated in Elasticsearch 8.x

# node.master: true

# node.data: true

# In Elasticsearch 8.x, a node is both a master-eligible and data node by default

# To change this behavior, use roles instead

#node.roles: [ master, data ]

# ----------------------------------- Paths ------------------------------------

path.data: /app/data/elasticsearch

# --------------------------------- Security ----------------------------------

xpack.security.enabled: true

# Transport layer settings (for node-to-node communication)

xpack.security.transport.ssl.enabled: false

# xpack.security.transport.ssl.verification_mode: none

# xpack.security.transport.ssl.keystore.path: /app/data/config/elastic-certificates.p12

# xpack.security.transport.ssl.truststore.path: /app/data/config/elastic-certificates.p12

# HTTP layer settings (for client connections)

xpack.security.http.ssl.enabled: false

# Allow basic auth

xpack.security.authc.token.enabled: false

xpack.security.authc.api_key.enabled: false

# Required password hashing algorithm

xpack.security.authc.password_hashing.algorithm: bcrypt

# ----------------------------------- Memory ----------------------------------

bootstrap.memory_lock: false

http.cors.allow-headers: "X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization"

action.auto_create_index: .security,.monitoring*,.watches,.triggered_watches,.watcher-history*,.ml* 

# ---------------------------------- Performance Optimizations ----------------------------------

# Circuit breaker settings

indices.breaker.total.limit: 70%

# Memory optimization

indices.fielddata.cache.size: 20%

indices.memory.index_buffer_size: 20%

# Thread pool and queue sizes

thread_pool.write.queue_size: 1000

thread_pool.search.queue_size: 1000

# Indexing settings

indices.queries.cache.size: 15%

bootstrap.memory_lock: true

# I/O optimization

bootstrap.system_call_filter: false

# Refresh interval - Set to a higher value if you prioritize indexing speed over search freshness

# indices.refresh_interval: 30s

# This setting should be applied per index, not globally

# Index defaults

index.number_of_shards: 1

index.number_of_replicas: 0

# Merge settings for better indexing performance

index.merge.scheduler.max_thread_count: 1 

index.merge.policy.floor_segment: 2mb

index.merge.policy.max_merge_at_once: 4

index.merge.policy.segments_per_tier: 8

# GC settings - these complement the JVM options set in start.sh

processors: ${PROCESSORS:1} 

 No newline at end of file

# Source environment variables

source /app/.env

# Function to generate a random password

generate_password() {

    cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1

}

# Create a secrets directory in data (which is writable)

mkdir -p /app/data/secrets

# Set the elastic user password - will be used throughout the script

ELASTIC_PASSWORD="cloudron"

echo "$ELASTIC_PASSWORD" > /app/data/secrets/elastic_password

# Set up the correct directories first

mkdir -p /app/data/elasticsearch

mkdir -p /app/data/logs

mkdir -p /app/data/logs/gc

mkdir -p /app/data/config

mkdir -p /app/data/run  # Directory for PID file

# Create a writable JDK directory for Elasticsearch FIRST

# This must be done before trying to use Java

mkdir -p /app/data/jdk/bin

if [ ! -L /app/data/jdk/bin/java ]; then

    echo "Setting up Java environment..."

    # First check if Java is available

    if [ ! -f /usr/lib/jvm/java-17-openjdk-amd64/bin/java ]; then

        echo "ERROR: Java is not found at expected location /usr/lib/jvm/java-17-openjdk-amd64/bin/java"

        # Try to find it elsewhere

        echo "Available Java installations:"

        find /usr -name java | grep -i jdk

        ls -la /usr/lib/jvm/

        echo "Will try default system Java instead"

        # Try to use default Java

        which java

        JAVA_PATH=$(which java)

        if [ -n "$JAVA_PATH" ]; then

            ln -sf $JAVA_PATH /app/data/jdk/bin/java

            ln -sf $(which javac 2>/dev/null) /app/data/jdk/bin/javac 2>/dev/null

            ln -sf $(which javadoc 2>/dev/null) /app/data/jdk/bin/javadoc 2>/dev/null

            ln -sf $(which jar 2>/dev/null) /app/data/jdk/bin/jar 2>/dev/null

        else

            echo "ERROR: No Java found on system. Elasticsearch requires Java to run."

            exit 1

        fi

    else

        # Create symbolic links from Java 17 to our writable JDK directory

        ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/java /app/data/jdk/bin/java

        ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/javac /app/data/jdk/bin/javac

        ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/javadoc /app/data/jdk/bin/javadoc

        ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/jar /app/data/jdk/bin/jar

    fi

    # Verify Java is available and linked correctly

    if [ -L /app/data/jdk/bin/java ]; then

        echo "Checking Java version:"

        /app/data/jdk/bin/java -version || echo "WARNING: Java is linked but not working!"

    else

        echo "ERROR: Failed to link Java executable"

    fi

    echo "Java environment setup complete"

fi

# Check for initialization status

if [[ ! -f /app/data/.initialized ]]; then

    echo "Fresh installation, initializing..."

    # Generate and store passwords

    if [ -z "$ELASTIC_PASSWORD" ]; then

        ELASTIC_PASSWORD=$(generate_password)

        echo "Generated new secure password for Elasticsearch user 'elastic'"

    # Create an initial keystore and add bootstrap password for first startup

    cd /usr/share/elasticsearch

    # Set config path for Elasticsearch

    export ES_PATH_CONF=/app/data/config

    # Ensure proper ownership of config directory

    chown -R elasticsearch:elasticsearch /app/data/config

    # Initialize the elasticsearch keystore if it doesn't exist

    if [ ! -f /app/data/config/elasticsearch.keystore ]; then

        echo "Creating Elasticsearch keystore..."

        # Create keystore as the elasticsearch user

        su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore create" elasticsearch

    else

        # If keystore exists but needs to be recreated (e.g. due to permission issues)

        echo "Keystore already exists. Ensuring proper permissions..."

        rm -f /app/data/config/elasticsearch.keystore

        su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore create" elasticsearch

    fi

    echo "$ELASTIC_PASSWORD" > /app/data/secrets/elastic_password

    # Add the bootstrap password to the keystore - this is needed for first startup

    echo "Setting bootstrap password..."

    echo "$ELASTIC_PASSWORD" | su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore add -f -x 'bootstrap.password' --stdin" elasticsearch

    # Mark as initialized

    touch /app/data/.initialized

    echo "Initialization complete."

else

    echo "Loading existing configuration..."

    # Load existing passwords

    if [ -f "/app/data/secrets/elastic_password" ]; then

        ELASTIC_PASSWORD=$(cat /app/data/secrets/elastic_password)

    # Re-add the bootstrap password to ensure it's set correctly

    cd /usr/share/elasticsearch

    # Set config path for Elasticsearch

    export ES_PATH_CONF=/app/data/config

    # Ensure elasticsearch user owns the keystore

    chown -R elasticsearch:elasticsearch /app/data/config

    echo "Updating bootstrap password in keystore..."

    echo "$ELASTIC_PASSWORD" | su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore add -f -x 'bootstrap.password' --stdin" elasticsearch

fi

# Copy the entire Elasticsearch config directory if not already set up

if [ ! -f /app/data/config/elasticsearch.yml ]; then

    echo "Copying Elasticsearch configuration files..."

    cp -r /usr/share/elasticsearch/config/* /app/data/config/ || true

    # Override with our custom elasticsearch.yml

    cp /app/elasticsearch.yml /app/data/config/elasticsearch.yml || true

    echo "Elasticsearch configuration files copied"

fi

# Set up the correct directories

mkdir -p /app/data/elasticsearch

mkdir -p /app/data/logs

mkdir -p /app/data/config

# Modify jvm.options to use a writable location for GC logs

if [ -f /app/data/config/jvm.options ]; then

    echo "Updating JVM options for GC logs..."

    # Create a temporary file with updated options

    cat /app/data/config/jvm.options | sed 's|logs/gc.log|/app/data/logs/gc/gc.log|g' > /tmp/jvm.options.new

    mv /tmp/jvm.options.new /app/data/config/jvm.options

    echo "JVM options updated"

fi

# Copy elasticsearch.yml to config directory if it doesn't exist

if [ ! -f /app/data/config/elasticsearch.yml ]; then

    cp /app/data/config/elasticsearch.yml.orig /app/data/config/elasticsearch.yml || true

    # If the original doesn't exist, copy from the one we bundled

    if [ ! -f /app/data/config/elasticsearch.yml ]; then

        cp /app/data/config/elasticsearch.yml.orig /app/data/config/elasticsearch.yml 2>/dev/null || cp /app/elasticsearch.yml /app/data/config/elasticsearch.yml 2>/dev/null || true

# Generate SSL certificates for Elasticsearch if they don't exist

if [ ! -f /app/data/config/elastic-certificates.p12 ]; then

    echo "Generating self-signed certificates for Elasticsearch transport layer..."

    # Create a temporary directory for certificate generation

    mkdir -p /tmp/elastic-certs

    # Run the elasticsearch-certutil command to generate the certificate

    cd /usr/share/elasticsearch

    ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-certutil ca \

        --out /tmp/elastic-certs/elastic-stack-ca.p12 \

        --pass "" \

        --silent

    # Generate the certificates for nodes

    ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-certutil cert \

        --ca /tmp/elastic-certs/elastic-stack-ca.p12 \

        --ca-pass "" \

        --out /app/data/config/elastic-certificates.p12 \

        --pass "" \

        --silent

    # Set proper permissions for the certificates

    chown elasticsearch:elasticsearch /app/data/config/elastic-certificates.p12

    chmod 600 /app/data/config/elastic-certificates.p12

    echo "SSL certificates generated successfully"

fi

# Create users file if it doesn't exist

if [ ! -f /app/data/config/users ]; then

    echo "Creating initial users file..."

    # Create users file with elastic user and password

    echo 'elastic:$2a$10$BtVRGAoL8AbgEKnlvYj8cewQF3QkUz1pyL.Ga3j.jFKNUk2yh7.zW' > /app/data/config/users

    echo 'kibana_system:$2a$10$BtVRGAoL8AbgEKnlvYj8cewQF3QkUz1pyL.Ga3j.jFKNUk2yh7.zW' >> /app/data/config/users

    # Set permissions for the users file

    chown elasticsearch:elasticsearch /app/data/config/users

    chmod 600 /app/data/config/users

    # Create users_roles file

    echo 'superuser:elastic' > /app/data/config/users_roles

    chown elasticsearch:elasticsearch /app/data/config/users_roles

    chmod 600 /app/data/config/users_roles

    echo "Users file created"

fi

# Add a special setting to elasticsearch.yml to disable TLS security for HTTP

echo "Adding security settings to elasticsearch.yml..."

if ! grep -q "xpack.security.http.ssl.enabled" /app/data/config/elasticsearch.yml; then

    echo "" >> /app/data/config/elasticsearch.yml

    echo "# Security settings for Elasticsearch 8.x" >> /app/data/config/elasticsearch.yml

    echo "xpack.security.enabled: true" >> /app/data/config/elasticsearch.yml

    echo "xpack.security.http.ssl.enabled: false" >> /app/data/config/elasticsearch.yml

    echo "xpack.security.transport.ssl.enabled: true" >> /app/data/config/elasticsearch.yml

    echo "xpack.security.transport.ssl.verification_mode: none" >> /app/data/config/elasticsearch.yml

    echo "xpack.security.authc.password_hashing.algorithm: bcrypt" >> /app/data/config/elasticsearch.yml

    echo "Security settings added to elasticsearch.yml"

fi

# Ensure network.host is set to 0.0.0.0 in elasticsearch.yml

if ! grep -q "network.host: 0.0.0.0" /app/data/config/elasticsearch.yml; then

    echo "" >> /app/data/config/elasticsearch.yml

    echo "# Set network host to all interfaces" >> /app/data/config/elasticsearch.yml

    echo "network.host: 0.0.0.0" >> /app/data/config/elasticsearch.yml

    echo "Network settings updated"

fi

# Ensure discovery.type is set to single-node

if ! grep -q "discovery.type: single-node" /app/data/config/elasticsearch.yml; then

    echo "" >> /app/data/config/elasticsearch.yml

    echo "# Single node setup" >> /app/data/config/elasticsearch.yml

    echo "discovery.type: single-node" >> /app/data/config/elasticsearch.yml

    echo "Discovery settings updated"

fi

# Ensure permissions are correct for writable directories

chown -R elasticsearch:elasticsearch /app/data/elasticsearch /app/data/logs /app/data/config /app/data/jdk /app/data/run /app/data/secrets

# Display network interfaces for debugging

ip a

# Enable system limits for elasticsearch

echo "Setting system limits for Elasticsearch..."

# Increase file descriptor limit

ulimit -n 65536 || echo "Warning: Could not set file descriptor limit"

# Allow memory locking

ulimit -l unlimited || echo "Warning: Could not set memory lock limit"

# Enable huge pages if available 

echo never > /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || true

# Set max map count if we have permission

sysctl -w vm.max_map_count=262144 2>/dev/null || echo "Warning: Could not set vm.max_map_count"

# Set required environment variables for Elasticsearch

export ES_HOME=/usr/share/elasticsearch

export ES_PATH_CONF=/app/data/config

# Calculate optimal heap size (50% of available memory, with 4GB default container limit)

# Default to 2GB if we can't determine container memory

CONTAINER_MEM=$(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo "4294967296")

if [ "$CONTAINER_MEM" = "max" ]; then

    CONTAINER_MEM="4294967296"  # Default to 4GB if unlimited

fi

HEAP_SIZE=$(expr $CONTAINER_MEM / 2097152)  # Convert to MB and take 50%

if [ $HEAP_SIZE -gt 31744 ]; then

    # Elasticsearch has a maximum recommended heap size of 31GB

    HEAP_SIZE=31744

fi

if [ $HEAP_SIZE -lt 512 ]; then

    # Minimum recommended is 512MB

    HEAP_SIZE=512

fi

# Set JVM options with calculated heap size

export ES_JAVA_OPTS="-Xms${HEAP_SIZE}m -Xmx${HEAP_SIZE}m"

# Add GC optimization flags

export ES_JAVA_OPTS="$ES_JAVA_OPTS -XX:+UseG1GC -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30"

export PATH=$ES_HOME/bin:$PATH

# Ensure permissions are correct

chown -R elasticsearch:elasticsearch /app/data/elasticsearch /app/data/logs /app/data/config

# Additional JVM options to redirect GC logs

export ES_JAVA_OPTS="$ES_JAVA_OPTS -Xlog:gc*,gc+age=trace,safepoint:file=/app/data/logs/gc/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m"

# Print the network interfaces for debugging

echo "Network interfaces:"

ip addr show

# Pre-create PID file with proper permissions

touch /app/data/run/elasticsearch.pid

chown elasticsearch:elasticsearch /app/data/run/elasticsearch.pid

# Modify the Elasticsearch startup script to use our linked Java and writable PID file location

# Include additional parameters for security settings, using compatible settings for 8.x

ES_START_CMD="ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch -E xpack.security.enabled=true -E bootstrap.password=$ELASTIC_PASSWORD -d -p /app/data/run/elasticsearch.pid"

# Start Elasticsearch in the background

echo "Starting Elasticsearch..."

cd /usr/share/elasticsearch

su -c "ES_PATH_CONF=/app/data/config /usr/share/elasticsearch/bin/elasticsearch -d -p /app/data/elasticsearch.pid" elasticsearch

# Wait for Elasticsearch to be up

# Execute Elasticsearch directly without using the elasticsearch-cli script

su -c "$ES_START_CMD" elasticsearch

# Wait for Elasticsearch to be up - checking for HTTP 200 response (without auth first)

echo "Waiting for Elasticsearch to start..."

attempts=0

max_attempts=60

until $(curl --output /dev/null --silent --head --fail http://localhost:9200); do

    # Also check if Elasticsearch is running - sometimes it may have crashed

    if ! ps -p $(cat /app/data/run/elasticsearch.pid 2>/dev/null) > /dev/null 2>&1; then

        echo "Elasticsearch process is not running. Check logs at /app/data/logs/"

        cat /app/data/logs/*.log

        exit 1

    fi

    printf '.'

    sleep 5

    if [ $attempts -ge $max_attempts ]; then

        echo "Elasticsearch failed to start after 5 minutes. Check logs at /app/data/logs/"

        cat /app/data/logs/*.log

        exit 1

    fi

done

echo "Elasticsearch is up and running!"

# Now that Elasticsearch is running, set the elastic user password

# Now that Elasticsearch is running, reset the elastic user password

echo "Setting elastic user password..."

cd /usr/share/elasticsearch

echo "y" | bin/elasticsearch-reset-password -u elastic -b -p "$ELASTIC_PASSWORD" --url "http://localhost:9200" || true

echo "y" | ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-reset-password -u elastic -b -p "$ELASTIC_PASSWORD" --url "http://localhost:9200" || true

# Test connection with the new password

echo "Testing connection with elastic user..."

if curl -s -u elastic:$ELASTIC_PASSWORD http://localhost:9200; then

    echo "Authentication successful!"

else

    echo "Warning: Authentication failed. Check logs for details."

fi

# Display the credentials

echo "-----------------------------"

echo "Authentication credentials:"

echo "  User: elastic"

echo "  Password: $ELASTIC_PASSWORD"

echo "  Note: Always use HTTP port 9200 for REST API connections"

echo "-----------------------------"

# Create a credentials file for reference