Add performance optimizations for Elasticsearch
This commit is contained in:
Andreas Düren
@@ -5,8 +5,12 @@ cluster.name: cloudron-cluster
|
||||
|
||||
# ------------------------------------ Node ------------------------------------
|
||||
node.name: ${HOSTNAME}
|
||||
node.master: true
|
||||
node.data: true
|
||||
# The following settings are deprecated in Elasticsearch 8.x
|
||||
# node.master: true
|
||||
# node.data: true
|
||||
# In Elasticsearch 8.x, a node is both a master-eligible and data node by default
|
||||
# To change this behavior, use roles instead
|
||||
#node.roles: [ master, data ]
|
||||
|
||||
# ----------------------------------- Paths ------------------------------------
|
||||
path.data: /app/data/elasticsearch
|
||||
@@ -22,9 +26,23 @@ discovery.type: single-node
|
||||
|
||||
# --------------------------------- Security ----------------------------------
|
||||
xpack.security.enabled: true
|
||||
|
||||
# Transport layer settings (for node-to-node communication)
|
||||
xpack.security.transport.ssl.enabled: false
|
||||
# xpack.security.transport.ssl.verification_mode: none
|
||||
# xpack.security.transport.ssl.keystore.path: /app/data/config/elastic-certificates.p12
|
||||
# xpack.security.transport.ssl.truststore.path: /app/data/config/elastic-certificates.p12
|
||||
|
||||
# HTTP layer settings (for client connections)
|
||||
xpack.security.http.ssl.enabled: false
|
||||
|
||||
# Allow basic auth
|
||||
xpack.security.authc.token.enabled: false
|
||||
xpack.security.authc.api_key.enabled: false
|
||||
|
||||
# Required password hashing algorithm
|
||||
xpack.security.authc.password_hashing.algorithm: bcrypt
|
||||
|
||||
# ----------------------------------- Memory ----------------------------------
|
||||
bootstrap.memory_lock: false
|
||||
|
||||
@@ -34,4 +52,41 @@ http.cors.allow-origin: "*"
|
||||
http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE
|
||||
http.cors.allow-headers: "X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization"
|
||||
|
||||
action.auto_create_index: .security,.monitoring*,.watches,.triggered_watches,.watcher-history*,.ml*
|
||||
action.auto_create_index: .security,.monitoring*,.watches,.triggered_watches,.watcher-history*,.ml*
|
||||
|
||||
# ---------------------------------- Performance Optimizations ----------------------------------
|
||||
|
||||
# Circuit breaker settings
|
||||
indices.breaker.total.limit: 70%
|
||||
|
||||
# Memory optimization
|
||||
indices.fielddata.cache.size: 20%
|
||||
indices.memory.index_buffer_size: 20%
|
||||
|
||||
# Thread pool and queue sizes
|
||||
thread_pool.write.queue_size: 1000
|
||||
thread_pool.search.queue_size: 1000
|
||||
|
||||
# Indexing settings
|
||||
indices.queries.cache.size: 15%
|
||||
bootstrap.memory_lock: true
|
||||
|
||||
# I/O optimization
|
||||
bootstrap.system_call_filter: false
|
||||
|
||||
# Refresh interval - Set to a higher value if you prioritize indexing speed over search freshness
|
||||
# indices.refresh_interval: 30s
|
||||
# This setting should be applied per index, not globally
|
||||
|
||||
# Index defaults
|
||||
index.number_of_shards: 1
|
||||
index.number_of_replicas: 0
|
||||
|
||||
# Merge settings for better indexing performance
|
||||
index.merge.scheduler.max_thread_count: 1
|
||||
index.merge.policy.floor_segment: 2mb
|
||||
index.merge.policy.max_merge_at_once: 4
|
||||
index.merge.policy.segments_per_tier: 8
|
||||
|
||||
# GC settings - these complement the JVM options set in start.sh
|
||||
processors: ${PROCESSORS:1}
|
||||
289
start.sh
289
start.sh
@@ -4,67 +4,278 @@ set -e
|
||||
# Source environment variables
|
||||
source /app/.env
|
||||
|
||||
# Function to generate a random password
|
||||
generate_password() {
|
||||
cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1
|
||||
}
|
||||
|
||||
# Create a secrets directory in data (which is writable)
|
||||
mkdir -p /app/data/secrets
|
||||
|
||||
# Set the elastic user password - will be used throughout the script
|
||||
ELASTIC_PASSWORD="cloudron"
|
||||
echo "$ELASTIC_PASSWORD" > /app/data/secrets/elastic_password
|
||||
|
||||
# Set up the correct directories first
|
||||
mkdir -p /app/data/elasticsearch
|
||||
mkdir -p /app/data/logs
|
||||
mkdir -p /app/data/logs/gc
|
||||
mkdir -p /app/data/config
|
||||
mkdir -p /app/data/run # Directory for PID file
|
||||
|
||||
# Create a writable JDK directory for Elasticsearch FIRST
|
||||
# This must be done before trying to use Java
|
||||
mkdir -p /app/data/jdk/bin
|
||||
if [ ! -L /app/data/jdk/bin/java ]; then
|
||||
echo "Setting up Java environment..."
|
||||
# First check if Java is available
|
||||
if [ ! -f /usr/lib/jvm/java-17-openjdk-amd64/bin/java ]; then
|
||||
echo "ERROR: Java is not found at expected location /usr/lib/jvm/java-17-openjdk-amd64/bin/java"
|
||||
# Try to find it elsewhere
|
||||
echo "Available Java installations:"
|
||||
find /usr -name java | grep -i jdk
|
||||
ls -la /usr/lib/jvm/
|
||||
echo "Will try default system Java instead"
|
||||
# Try to use default Java
|
||||
which java
|
||||
JAVA_PATH=$(which java)
|
||||
if [ -n "$JAVA_PATH" ]; then
|
||||
ln -sf $JAVA_PATH /app/data/jdk/bin/java
|
||||
ln -sf $(which javac 2>/dev/null) /app/data/jdk/bin/javac 2>/dev/null
|
||||
ln -sf $(which javadoc 2>/dev/null) /app/data/jdk/bin/javadoc 2>/dev/null
|
||||
ln -sf $(which jar 2>/dev/null) /app/data/jdk/bin/jar 2>/dev/null
|
||||
else
|
||||
echo "ERROR: No Java found on system. Elasticsearch requires Java to run."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
# Create symbolic links from Java 17 to our writable JDK directory
|
||||
ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/java /app/data/jdk/bin/java
|
||||
ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/javac /app/data/jdk/bin/javac
|
||||
ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/javadoc /app/data/jdk/bin/javadoc
|
||||
ln -sf /usr/lib/jvm/java-17-openjdk-amd64/bin/jar /app/data/jdk/bin/jar
|
||||
fi
|
||||
|
||||
# Verify Java is available and linked correctly
|
||||
if [ -L /app/data/jdk/bin/java ]; then
|
||||
echo "Checking Java version:"
|
||||
/app/data/jdk/bin/java -version || echo "WARNING: Java is linked but not working!"
|
||||
else
|
||||
echo "ERROR: Failed to link Java executable"
|
||||
fi
|
||||
|
||||
echo "Java environment setup complete"
|
||||
fi
|
||||
|
||||
# Check for initialization status
|
||||
if [[ ! -f /app/data/.initialized ]]; then
|
||||
echo "Fresh installation, initializing..."
|
||||
|
||||
# Generate and store passwords
|
||||
if [ -z "$ELASTIC_PASSWORD" ]; then
|
||||
ELASTIC_PASSWORD=$(generate_password)
|
||||
echo "Generated new secure password for Elasticsearch user 'elastic'"
|
||||
# Create an initial keystore and add bootstrap password for first startup
|
||||
cd /usr/share/elasticsearch
|
||||
# Set config path for Elasticsearch
|
||||
export ES_PATH_CONF=/app/data/config
|
||||
|
||||
# Ensure proper ownership of config directory
|
||||
chown -R elasticsearch:elasticsearch /app/data/config
|
||||
|
||||
# Initialize the elasticsearch keystore if it doesn't exist
|
||||
if [ ! -f /app/data/config/elasticsearch.keystore ]; then
|
||||
echo "Creating Elasticsearch keystore..."
|
||||
# Create keystore as the elasticsearch user
|
||||
su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore create" elasticsearch
|
||||
else
|
||||
# If keystore exists but needs to be recreated (e.g. due to permission issues)
|
||||
echo "Keystore already exists. Ensuring proper permissions..."
|
||||
rm -f /app/data/config/elasticsearch.keystore
|
||||
su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore create" elasticsearch
|
||||
fi
|
||||
echo "$ELASTIC_PASSWORD" > /app/data/secrets/elastic_password
|
||||
|
||||
# Add the bootstrap password to the keystore - this is needed for first startup
|
||||
echo "Setting bootstrap password..."
|
||||
echo "$ELASTIC_PASSWORD" | su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore add -f -x 'bootstrap.password' --stdin" elasticsearch
|
||||
|
||||
# Mark as initialized
|
||||
touch /app/data/.initialized
|
||||
echo "Initialization complete."
|
||||
else
|
||||
echo "Loading existing configuration..."
|
||||
# Load existing passwords
|
||||
if [ -f "/app/data/secrets/elastic_password" ]; then
|
||||
ELASTIC_PASSWORD=$(cat /app/data/secrets/elastic_password)
|
||||
fi
|
||||
# Re-add the bootstrap password to ensure it's set correctly
|
||||
cd /usr/share/elasticsearch
|
||||
# Set config path for Elasticsearch
|
||||
export ES_PATH_CONF=/app/data/config
|
||||
|
||||
# Ensure elasticsearch user owns the keystore
|
||||
chown -R elasticsearch:elasticsearch /app/data/config
|
||||
|
||||
echo "Updating bootstrap password in keystore..."
|
||||
echo "$ELASTIC_PASSWORD" | su -c "ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch-keystore add -f -x 'bootstrap.password' --stdin" elasticsearch
|
||||
fi
|
||||
|
||||
# Set up the correct directories
|
||||
mkdir -p /app/data/elasticsearch
|
||||
mkdir -p /app/data/logs
|
||||
mkdir -p /app/data/config
|
||||
|
||||
# Copy elasticsearch.yml to config directory if it doesn't exist
|
||||
# Copy the entire Elasticsearch config directory if not already set up
|
||||
if [ ! -f /app/data/config/elasticsearch.yml ]; then
|
||||
cp /app/data/config/elasticsearch.yml.orig /app/data/config/elasticsearch.yml || true
|
||||
# If the original doesn't exist, copy from the one we bundled
|
||||
if [ ! -f /app/data/config/elasticsearch.yml ]; then
|
||||
cp /app/data/config/elasticsearch.yml.orig /app/data/config/elasticsearch.yml 2>/dev/null || cp /app/elasticsearch.yml /app/data/config/elasticsearch.yml 2>/dev/null || true
|
||||
fi
|
||||
echo "Copying Elasticsearch configuration files..."
|
||||
cp -r /usr/share/elasticsearch/config/* /app/data/config/ || true
|
||||
# Override with our custom elasticsearch.yml
|
||||
cp /app/elasticsearch.yml /app/data/config/elasticsearch.yml || true
|
||||
echo "Elasticsearch configuration files copied"
|
||||
fi
|
||||
|
||||
# Ensure permissions are correct
|
||||
chown -R elasticsearch:elasticsearch /app/data/elasticsearch /app/data/logs /app/data/config
|
||||
# Modify jvm.options to use a writable location for GC logs
|
||||
if [ -f /app/data/config/jvm.options ]; then
|
||||
echo "Updating JVM options for GC logs..."
|
||||
# Create a temporary file with updated options
|
||||
cat /app/data/config/jvm.options | sed 's|logs/gc.log|/app/data/logs/gc/gc.log|g' > /tmp/jvm.options.new
|
||||
mv /tmp/jvm.options.new /app/data/config/jvm.options
|
||||
echo "JVM options updated"
|
||||
fi
|
||||
|
||||
# Print the network interfaces for debugging
|
||||
echo "Network interfaces:"
|
||||
ip addr show
|
||||
# Generate SSL certificates for Elasticsearch if they don't exist
|
||||
if [ ! -f /app/data/config/elastic-certificates.p12 ]; then
|
||||
echo "Generating self-signed certificates for Elasticsearch transport layer..."
|
||||
|
||||
# Create a temporary directory for certificate generation
|
||||
mkdir -p /tmp/elastic-certs
|
||||
|
||||
# Run the elasticsearch-certutil command to generate the certificate
|
||||
cd /usr/share/elasticsearch
|
||||
ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-certutil ca \
|
||||
--out /tmp/elastic-certs/elastic-stack-ca.p12 \
|
||||
--pass "" \
|
||||
--silent
|
||||
|
||||
# Generate the certificates for nodes
|
||||
ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-certutil cert \
|
||||
--ca /tmp/elastic-certs/elastic-stack-ca.p12 \
|
||||
--ca-pass "" \
|
||||
--out /app/data/config/elastic-certificates.p12 \
|
||||
--pass "" \
|
||||
--silent
|
||||
|
||||
# Set proper permissions for the certificates
|
||||
chown elasticsearch:elasticsearch /app/data/config/elastic-certificates.p12
|
||||
chmod 600 /app/data/config/elastic-certificates.p12
|
||||
|
||||
echo "SSL certificates generated successfully"
|
||||
fi
|
||||
|
||||
# Create users file if it doesn't exist
|
||||
if [ ! -f /app/data/config/users ]; then
|
||||
echo "Creating initial users file..."
|
||||
# Create users file with elastic user and password
|
||||
echo 'elastic:$2a$10$BtVRGAoL8AbgEKnlvYj8cewQF3QkUz1pyL.Ga3j.jFKNUk2yh7.zW' > /app/data/config/users
|
||||
echo 'kibana_system:$2a$10$BtVRGAoL8AbgEKnlvYj8cewQF3QkUz1pyL.Ga3j.jFKNUk2yh7.zW' >> /app/data/config/users
|
||||
|
||||
# Set permissions for the users file
|
||||
chown elasticsearch:elasticsearch /app/data/config/users
|
||||
chmod 600 /app/data/config/users
|
||||
|
||||
# Create users_roles file
|
||||
echo 'superuser:elastic' > /app/data/config/users_roles
|
||||
chown elasticsearch:elasticsearch /app/data/config/users_roles
|
||||
chmod 600 /app/data/config/users_roles
|
||||
|
||||
echo "Users file created"
|
||||
fi
|
||||
|
||||
# Add a special setting to elasticsearch.yml to disable TLS security for HTTP
|
||||
echo "Adding security settings to elasticsearch.yml..."
|
||||
if ! grep -q "xpack.security.http.ssl.enabled" /app/data/config/elasticsearch.yml; then
|
||||
echo "" >> /app/data/config/elasticsearch.yml
|
||||
echo "# Security settings for Elasticsearch 8.x" >> /app/data/config/elasticsearch.yml
|
||||
echo "xpack.security.enabled: true" >> /app/data/config/elasticsearch.yml
|
||||
echo "xpack.security.http.ssl.enabled: false" >> /app/data/config/elasticsearch.yml
|
||||
echo "xpack.security.transport.ssl.enabled: true" >> /app/data/config/elasticsearch.yml
|
||||
echo "xpack.security.transport.ssl.verification_mode: none" >> /app/data/config/elasticsearch.yml
|
||||
echo "xpack.security.authc.password_hashing.algorithm: bcrypt" >> /app/data/config/elasticsearch.yml
|
||||
echo "Security settings added to elasticsearch.yml"
|
||||
fi
|
||||
|
||||
# Ensure network.host is set to 0.0.0.0 in elasticsearch.yml
|
||||
if ! grep -q "network.host: 0.0.0.0" /app/data/config/elasticsearch.yml; then
|
||||
echo "" >> /app/data/config/elasticsearch.yml
|
||||
echo "# Set network host to all interfaces" >> /app/data/config/elasticsearch.yml
|
||||
echo "network.host: 0.0.0.0" >> /app/data/config/elasticsearch.yml
|
||||
echo "Network settings updated"
|
||||
fi
|
||||
|
||||
# Ensure discovery.type is set to single-node
|
||||
if ! grep -q "discovery.type: single-node" /app/data/config/elasticsearch.yml; then
|
||||
echo "" >> /app/data/config/elasticsearch.yml
|
||||
echo "# Single node setup" >> /app/data/config/elasticsearch.yml
|
||||
echo "discovery.type: single-node" >> /app/data/config/elasticsearch.yml
|
||||
echo "Discovery settings updated"
|
||||
fi
|
||||
|
||||
# Ensure permissions are correct for writable directories
|
||||
chown -R elasticsearch:elasticsearch /app/data/elasticsearch /app/data/logs /app/data/config /app/data/jdk /app/data/run /app/data/secrets
|
||||
|
||||
# Display network interfaces for debugging
|
||||
ip a
|
||||
|
||||
# Enable system limits for elasticsearch
|
||||
echo "Setting system limits for Elasticsearch..."
|
||||
# Increase file descriptor limit
|
||||
ulimit -n 65536 || echo "Warning: Could not set file descriptor limit"
|
||||
# Allow memory locking
|
||||
ulimit -l unlimited || echo "Warning: Could not set memory lock limit"
|
||||
# Enable huge pages if available
|
||||
echo never > /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || true
|
||||
# Set max map count if we have permission
|
||||
sysctl -w vm.max_map_count=262144 2>/dev/null || echo "Warning: Could not set vm.max_map_count"
|
||||
|
||||
# Set required environment variables for Elasticsearch
|
||||
export ES_HOME=/usr/share/elasticsearch
|
||||
export ES_PATH_CONF=/app/data/config
|
||||
|
||||
# Calculate optimal heap size (50% of available memory, with 4GB default container limit)
|
||||
# Default to 2GB if we can't determine container memory
|
||||
CONTAINER_MEM=$(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo "4294967296")
|
||||
if [ "$CONTAINER_MEM" = "max" ]; then
|
||||
CONTAINER_MEM="4294967296" # Default to 4GB if unlimited
|
||||
fi
|
||||
HEAP_SIZE=$(expr $CONTAINER_MEM / 2097152) # Convert to MB and take 50%
|
||||
if [ $HEAP_SIZE -gt 31744 ]; then
|
||||
# Elasticsearch has a maximum recommended heap size of 31GB
|
||||
HEAP_SIZE=31744
|
||||
fi
|
||||
if [ $HEAP_SIZE -lt 512 ]; then
|
||||
# Minimum recommended is 512MB
|
||||
HEAP_SIZE=512
|
||||
fi
|
||||
|
||||
# Set JVM options with calculated heap size
|
||||
export ES_JAVA_OPTS="-Xms${HEAP_SIZE}m -Xmx${HEAP_SIZE}m"
|
||||
|
||||
# Add GC optimization flags
|
||||
export ES_JAVA_OPTS="$ES_JAVA_OPTS -XX:+UseG1GC -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30"
|
||||
export PATH=$ES_HOME/bin:$PATH
|
||||
|
||||
# Additional JVM options to redirect GC logs
|
||||
export ES_JAVA_OPTS="$ES_JAVA_OPTS -Xlog:gc*,gc+age=trace,safepoint:file=/app/data/logs/gc/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m"
|
||||
|
||||
# Pre-create PID file with proper permissions
|
||||
touch /app/data/run/elasticsearch.pid
|
||||
chown elasticsearch:elasticsearch /app/data/run/elasticsearch.pid
|
||||
|
||||
# Modify the Elasticsearch startup script to use our linked Java and writable PID file location
|
||||
# Include additional parameters for security settings, using compatible settings for 8.x
|
||||
ES_START_CMD="ES_PATH_CONF=/app/data/config ES_JAVA_HOME=/app/data/jdk /usr/share/elasticsearch/bin/elasticsearch -E xpack.security.enabled=true -E bootstrap.password=$ELASTIC_PASSWORD -d -p /app/data/run/elasticsearch.pid"
|
||||
|
||||
# Start Elasticsearch in the background
|
||||
echo "Starting Elasticsearch..."
|
||||
cd /usr/share/elasticsearch
|
||||
su -c "ES_PATH_CONF=/app/data/config /usr/share/elasticsearch/bin/elasticsearch -d -p /app/data/elasticsearch.pid" elasticsearch
|
||||
|
||||
# Wait for Elasticsearch to be up
|
||||
# Execute Elasticsearch directly without using the elasticsearch-cli script
|
||||
su -c "$ES_START_CMD" elasticsearch
|
||||
|
||||
# Wait for Elasticsearch to be up - checking for HTTP 200 response (without auth first)
|
||||
echo "Waiting for Elasticsearch to start..."
|
||||
attempts=0
|
||||
max_attempts=60
|
||||
until $(curl --output /dev/null --silent --head --fail http://localhost:9200); do
|
||||
# Also check if Elasticsearch is running - sometimes it may have crashed
|
||||
if ! ps -p $(cat /app/data/run/elasticsearch.pid 2>/dev/null) > /dev/null 2>&1; then
|
||||
echo "Elasticsearch process is not running. Check logs at /app/data/logs/"
|
||||
cat /app/data/logs/*.log
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '.'
|
||||
sleep 5
|
||||
|
||||
@@ -72,15 +283,24 @@ until $(curl --output /dev/null --silent --head --fail http://localhost:9200); d
|
||||
|
||||
if [ $attempts -ge $max_attempts ]; then
|
||||
echo "Elasticsearch failed to start after 5 minutes. Check logs at /app/data/logs/"
|
||||
cat /app/data/logs/*.log
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "Elasticsearch is up and running!"
|
||||
|
||||
# Now that Elasticsearch is running, set the elastic user password
|
||||
# Now that Elasticsearch is running, reset the elastic user password
|
||||
echo "Setting elastic user password..."
|
||||
cd /usr/share/elasticsearch
|
||||
echo "y" | bin/elasticsearch-reset-password -u elastic -b -p "$ELASTIC_PASSWORD" --url "http://localhost:9200" || true
|
||||
echo "y" | ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-reset-password -u elastic -b -p "$ELASTIC_PASSWORD" --url "http://localhost:9200" || true
|
||||
|
||||
# Test connection with the new password
|
||||
echo "Testing connection with elastic user..."
|
||||
if curl -s -u elastic:$ELASTIC_PASSWORD http://localhost:9200; then
|
||||
echo "Authentication successful!"
|
||||
else
|
||||
echo "Warning: Authentication failed. Check logs for details."
|
||||
fi
|
||||
|
||||
# Display the credentials
|
||||
echo "-----------------------------"
|
||||
@@ -90,6 +310,7 @@ echo ""
|
||||
echo "Authentication credentials:"
|
||||
echo " User: elastic"
|
||||
echo " Password: $ELASTIC_PASSWORD"
|
||||
echo " Note: Always use HTTP port 9200 for REST API connections"
|
||||
echo "-----------------------------"
|
||||
|
||||
# Create a credentials file for reference
|
||||
|
||||
Reference in New Issue
Block a user