Fix unsupported settings in elasticsearch.yml - Remove processors and bootstrap.system_call_filter settings

elasticsearch.yml

@@ -55,15 +55,11 @@ indices.queries.cache.size: 15%
 thread_pool.write.queue_size: 1000
 thread_pool.search.queue_size: 1000
 
-# I/O optimization
-bootstrap.system_call_filter: false
-
 # ---------------------------------- Index Templates ---------------------------
-# Instead of setting index settings directly (which is not allowed), 
+# IMPORTANT: Index-level settings MUST NOT be placed in this configuration file.
-# we configure index templates that will be applied to new indices.
+# These settings are now applied using an index template in the start.sh script.
-# Elasticsearch will automatically apply these templates to new indices.
+# 
-
+# The following settings have been removed and will be applied via template:
-# The following settings MUST be removed from this file:
 # - index.number_of_shards
 # - index.number_of_replicas
 # - index.merge.scheduler.max_thread_count
@@ -71,5 +67,5 @@ bootstrap.system_call_filter: false
 # - index.merge.policy.max_merge_at_once
 # - index.merge.policy.segments_per_tier
 
-# Set processors based on container configuration
+# The 'processors' setting has been removed as it's no longer supported in Elasticsearch 8.x
-processors: ${PROCESSORS:1} 
+# processors: ${PROCESSORS:1} 

start.sh

@@ -127,6 +127,36 @@ configure_elasticsearch() {
         chown -R elasticsearch:elasticsearch $ES_PATH_CONF
     fi
     
+    # CRITICAL FIX: Remove any index-level settings from elasticsearch.yml to prevent startup failure
+    if [ -f $ES_PATH_CONF/elasticsearch.yml ]; then
+        echo "Checking elasticsearch.yml for index-level settings..."
+        # Create a temporary file
+        TEMP_FILE=$(mktemp)
+        
+        # Filter out any index.* settings
+        grep -v "^index\." $ES_PATH_CONF/elasticsearch.yml > $TEMP_FILE
+        
+        # Also remove other known problematic settings for 8.x
+        grep -v "^processors:" $TEMP_FILE > $TEMP_FILE.2 && mv $TEMP_FILE.2 $TEMP_FILE
+        grep -v "^bootstrap.system_call_filter:" $TEMP_FILE > $TEMP_FILE.2 && mv $TEMP_FILE.2 $TEMP_FILE
+        
+        # Add warning comment
+        echo "" >> $TEMP_FILE
+        echo "# NOTE: The following settings have been removed from this file:" >> $TEMP_FILE
+        echo "# - All index.* settings (applied via index templates)" >> $TEMP_FILE
+        echo "# - processors setting (no longer supported in 8.x)" >> $TEMP_FILE
+        echo "# - bootstrap.system_call_filter (no longer supported in 8.x)" >> $TEMP_FILE
+        echo "# See the create_index_template function in start.sh for details" >> $TEMP_FILE
+        
+        # Replace the original file
+        cat $TEMP_FILE > $ES_PATH_CONF/elasticsearch.yml
+        rm $TEMP_FILE
+        
+        # Ensure proper ownership
+        chown elasticsearch:elasticsearch $ES_PATH_CONF/elasticsearch.yml
+        echo "Cleaned elasticsearch.yml configuration file"
+    fi
+    
     # Update JVM options for GC logs
     if [ -f $ES_PATH_CONF/jvm.options ]; then
         echo "Updating JVM options..."
@@ -189,9 +219,9 @@ create_index_template() {
     echo "Creating default index template with optimized settings..."
     
     # Wait a moment to ensure Elasticsearch is fully operational
-    sleep 5
+    sleep 10
     
-    # Define the template JSON
+    # Define the template JSON with all the index settings that were previously in elasticsearch.yml
     template_json=$(cat <<EOF
 {
   "index_patterns": ["*"],
@@ -219,17 +249,36 @@ create_index_template() {
 EOF
 )
     
-    # Apply the template
+    # Try multiple times in case Elasticsearch is still initializing
-    curl -s -X PUT "http://localhost:9200/_index_template/cloudron_defaults" \
+    for i in {1..5}; do
-      -H "Content-Type: application/json" \
+        echo "Attempt $i to create index template..."
-      -u "elastic:$ELASTIC_PASSWORD" \
+        
-      -d "$template_json" > /dev/null
+        # Apply the template
-      
+        response=$(curl -s -w "\n%{http_code}" -X PUT "http://localhost:9200/_index_template/cloudron_defaults" \
-    if [ $? -eq 0 ]; then
+          -H "Content-Type: application/json" \
-        echo "Index template created successfully."
+          -u "elastic:$ELASTIC_PASSWORD" \
-    else
+          -d "$template_json")
-        echo "Warning: Failed to create index template. Default settings may not be applied to new indices."
+          
-    fi
+        http_code=$(echo "$response" | tail -n1)
+        response_body=$(echo "$response" | sed '$d')
+        
+        if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
+            echo "✅ Index template created successfully with HTTP code $http_code"
+            echo "Response: $response_body"
+            return 0
+        else
+            echo "⚠️ Failed to create index template on attempt $i. HTTP code: $http_code"
+            echo "Response: $response_body"
+            sleep 5
+        fi
+    done
+    
+    echo "⚠️ Warning: Failed to create index template after multiple attempts."
+    echo "Default settings may not be applied to new indices, but Elasticsearch will still function."
+    echo "You can manually create the template later using Elasticsearch API."
+    
+    # Don't fail the startup process if template creation fails
+    return 0
 }
 
 # Set system limits - be more tolerant of container restrictions
@@ -303,7 +352,7 @@ start_elasticsearch() {
     echo "Waiting for Elasticsearch to start..."
     attempts=0
     max_attempts=60
-    until $(curl --output /dev/null --silent --head --fail http://localhost:9200); do
+    until $(curl --output /dev/null --silent --head --fail -u "elastic:$ELASTIC_PASSWORD" http://localhost:9200); do
         if ! ps -p $(cat /app/data/run/elasticsearch.pid 2>/dev/null) > /dev/null 2>&1; then
             echo "ERROR: Elasticsearch process is not running. Logs:"
             cat /app/data/logs/*.log
@@ -327,6 +376,7 @@ start_elasticsearch() {
     echo "y" | ES_JAVA_HOME=/app/data/jdk bin/elasticsearch-reset-password -u elastic -b -p "$ELASTIC_PASSWORD" --url "http://localhost:9200" || true
     
     # Create index template with the settings we removed from elasticsearch.yml
+    echo "Applying index templates with settings removed from elasticsearch.yml..."
     create_index_template
     
     # Create credentials file