Enable built-in indexer tables and buddy

2025-11-16 07:18:11 -06:00
parent 63a730a940
commit be0ec13d3e
9 changed files with 210 additions and 11 deletions
--- a/BUILD.md
+++ b/BUILD.md
@@ -13,14 +13,14 @@ cloudron build \
  --set-build-service builder.docker.due.ren \
  --build-service-token e3265de06b1d0e7bb38400539012a8433a74c2c96a17955e \
  --set-repository andreasdueren/affine-cloudron \
-  --tag 0.25.3
+  --tag 0.25.23
 ```

 ## Deployment Steps
 1. Remove any previous dev install of AFFiNE on the Cloudron (always reinstall from scratch).
 2. Install the freshly built image:
   ```bash
-   cloudron install --location affine.due.ren --image andreasdueren/affine-cloudron:0.25.3
+cloudron install --location affine.due.ren --image andreasdueren/affine-cloudron:0.25.23
   ```
 3. When prompted, confirm the app info and wait for Cloudron to report success (abort after ~30 seconds if installation stalls or errors to avoid hanging sessions).
 4. Visit `https://affine.due.ren` (or the chosen location) and sign in using Cloudron SSO.
--- a/CloudronManifest.json
+++ b/CloudronManifest.json
@@ -5,8 +5,8 @@
  "description": "Next-gen knowledge base that blends docs, whiteboards, and databases for self-hosted teams.",
  "website": "https://affine.pro",
  "contactEmail": "support@affine.pro",
-  "version": "0.25.3",
-  "changelog": "Upgrade upstream AFFiNE runtime to v0.25.3 and keep Cloudron OIDC wiring",
+  "version": "0.25.23",
+  "changelog": "Stop printing wrapper banner so buddy stdout stays parseable for searchd",
  "icon": "file://icon.png",
  "manifestVersion": 2,
  "minBoxVersion": "7.0.0",
--- a/21
+++ b/21
@@ -14,9 +14,21 @@ ENV APP_CODE_DIR=/app/code \

 RUN mkdir -p "$APP_CODE_DIR" "$APP_DATA_DIR" "$APP_RUNTIME_DIR" "$APP_TMP_DIR" && \
    apt-get update && \
-    apt-get install -y --no-install-recommends jq python3 ca-certificates curl openssl libjemalloc2 && \
+    apt-get install -y --no-install-recommends jq python3 ca-certificates curl openssl libjemalloc2 postgresql-client && \
    rm -rf /var/lib/apt/lists/*

+RUN curl -fsSL https://repo.manticoresearch.com/GPG-KEY-manticore > /tmp/manticore.key && \
+    curl -fsSL https://repo.manticoresearch.com/GPG-KEY-SHA256-manticore >> /tmp/manticore.key && \
+    gpg --dearmor -o /usr/share/keyrings/manticore.gpg /tmp/manticore.key && \
+    rm /tmp/manticore.key && \
+    echo "deb [signed-by=/usr/share/keyrings/manticore.gpg] https://repo.manticoresearch.com/repository/manticoresearch_jammy/ jammy main" > /etc/apt/sources.list.d/manticore.list && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends manticore manticore-extra && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN ln -sf /usr/share/manticore/modules/manticore-buddy/bin/manticore-buddy /usr/bin/manticore-buddy
+RUN chown -R cloudron:cloudron /usr/share/manticore
+
 # bring in the upstream runtime and packaged server artifacts
 COPY --from=upstream /usr/local /usr/local
 COPY --from=upstream /opt /opt
@@ -25,13 +37,16 @@ COPY --from=upstream /app "$APP_BUILD_DIR"
 # configuration, launch scripts, and defaults
 COPY start.sh "$APP_CODE_DIR/start.sh"
 COPY run-affine.sh "$APP_CODE_DIR/run-affine.sh"
+COPY run-manticore.sh "$APP_CODE_DIR/run-manticore.sh"
+COPY run-buddy.sh "$APP_CODE_DIR/run-buddy.sh"
 COPY nginx.conf "$APP_CODE_DIR/nginx.conf"
 COPY supervisord.conf "$APP_CODE_DIR/supervisord.conf"
 COPY config.example.json "$APP_CODE_DIR/config.example.json"
 COPY tmp_data/ "$APP_TMP_DIR/"
+COPY manticore/ "$APP_CODE_DIR/manticore/"

-RUN chmod +x "$APP_CODE_DIR/start.sh" "$APP_CODE_DIR/run-affine.sh" && \
-    chown cloudron:cloudron "$APP_CODE_DIR/start.sh" "$APP_CODE_DIR/run-affine.sh" && \
+RUN chmod +x "$APP_CODE_DIR/start.sh" "$APP_CODE_DIR/run-affine.sh" "$APP_CODE_DIR/run-manticore.sh" "$APP_CODE_DIR/run-buddy.sh" && \
+    chown cloudron:cloudron "$APP_CODE_DIR/start.sh" "$APP_CODE_DIR/run-affine.sh" "$APP_CODE_DIR/run-manticore.sh" "$APP_CODE_DIR/run-buddy.sh" && \
    chown -R cloudron:cloudron "$APP_DATA_DIR" "$APP_RUNTIME_DIR" "$APP_TMP_DIR"

 EXPOSE 3000
--- a/manticore/manticore.conf
+++ b/manticore/manticore.conf
@@ -0,0 +1,23 @@
+#
+# Manticore Search configuration for AFFiNE on Cloudron.
+# Keeps all runtime state under /app/data so backups include the indexer data.
+#
+
+common {
+    plugin_dir = /app/data/manticore/plugins
+}
+
+searchd {
+    listen = 127.0.0.1:9306:mysql41
+    listen = 127.0.0.1:9308:http
+    listen = 127.0.0.1:9312
+    listen = /run/manticore/manticore.sock:mysql41
+
+    log = /app/data/manticore/logs/searchd.log
+    query_log = /app/data/manticore/logs/query.log
+    binlog_path = /app/data/manticore/binlog
+    data_dir = /app/data/manticore/data
+    pid_file = /run/manticore/searchd.pid
+    mysql_version_string = 8.0.33
+    buddy_path = /app/code/run-buddy.sh --disable-telemetry
+}
--- a/run-affine.sh
+++ b/run-affine.sh
@@ -89,7 +89,9 @@ PY
  else
    export AFFINE_SERVER_HTTPS=false
  fi
-  export AFFINE_INDEXER_ENABLED="${AFFINE_INDEXER_ENABLED:-false}"
+  export AFFINE_INDEXER_ENABLED="${AFFINE_INDEXER_ENABLED:-true}"
+  export AFFINE_INDEXER_SEARCH_PROVIDER="${AFFINE_INDEXER_SEARCH_PROVIDER:-manticoresearch}"
+  export AFFINE_INDEXER_SEARCH_ENDPOINT="${AFFINE_INDEXER_SEARCH_ENDPOINT:-http://127.0.0.1:9308}"
 }

 ensure_runtime_envs() {
@@ -99,6 +101,45 @@ ensure_runtime_envs() {
  ensure_server_env
 }

+# Helper to parse indexer endpoint into host/port for readiness checks
+wait_for_indexer() {
+  if [ "${AFFINE_INDEXER_ENABLED:-false}" != "true" ]; then
+    return
+  fi
+  local endpoint="${AFFINE_INDEXER_SEARCH_ENDPOINT:-}"
+  if [ -z "$endpoint" ]; then
+    return
+  fi
+  log "Waiting for indexer endpoint ${endpoint}"
+  if python3 - "$endpoint" <<'PY'; then
+import socket
+import sys
+import time
+from urllib.parse import urlparse
+
+endpoint = sys.argv[1]
+if not endpoint.startswith(('http://', 'https://')):
+    endpoint = 'http://' + endpoint
+parsed = urlparse(endpoint)
+host = parsed.hostname
+port = parsed.port or (443 if parsed.scheme == 'https' else 80)
+if not host or not port:
+    sys.exit(1)
+
+for _ in range(60):
+    try:
+        with socket.create_connection((host, port), timeout=2):
+            sys.exit(0)
+    except OSError:
+        time.sleep(1)
+sys.exit(1)
+PY
+    log "Indexer is ready"
+  else
+    log "Indexer at ${endpoint} not reachable after waiting, continuing startup"
+  fi
+}
+
 patch_upload_limits() {
  local target="$APP_DIR/dist/main.js"
  if [ ! -f "$target" ]; then
@@ -228,6 +269,7 @@ NODE

 log "Running AFFiNE pre-deployment migrations"
 ensure_runtime_envs
+wait_for_indexer
 node ./scripts/self-host-predeploy.js
 patch_upload_limits
 grant_team_plan_features
--- a/run-buddy.sh
+++ b/run-buddy.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -euo pipefail
+
+ENV_EXPORT_FILE=${ENV_EXPORT_FILE:-/run/affine/runtime.env}
+
+if [ -f "$ENV_EXPORT_FILE" ]; then
+  set -a
+  # shellcheck disable=SC1090
+  source "$ENV_EXPORT_FILE"
+  set +a
+fi
+
+exec /usr/bin/manticore-buddy "$@"
--- a/run-manticore.sh
+++ b/run-manticore.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -euo pipefail
+
+MANTICORE_CONF=${MANTICORE_CONF:-/app/data/manticore/manticore.conf}
+MANTICORE_RUN_DIR=${MANTICORE_RUN_DIR:-/run/manticore}
+
+mkdir -p "$MANTICORE_RUN_DIR"
+rm -f "$MANTICORE_RUN_DIR/searchd.pid"
+
+exec /usr/bin/searchd --nodetach -c "$MANTICORE_CONF"
--- a/start.sh
+++ b/start.sh
@@ -9,7 +9,11 @@ APP_BUILD_DIR=${APP_BUILD_DIR:-/app/code/affine}
 APP_HOME_DIR=${APP_HOME_DIR:-/app/data/home}
 AFFINE_HOME=${AFFINE_HOME:-$APP_HOME_DIR/.affine}
 ENV_EXPORT_FILE=${ENV_EXPORT_FILE:-$APP_RUNTIME_DIR/runtime.env}
-export APP_CODE_DIR APP_DATA_DIR APP_RUNTIME_DIR APP_TMP_DIR APP_BUILD_DIR APP_HOME_DIR AFFINE_HOME ENV_EXPORT_FILE
+MANTICORE_DATA_DIR=${MANTICORE_DATA_DIR:-$APP_DATA_DIR/manticore}
+MANTICORE_CONFIG_FILE=${MANTICORE_CONFIG_FILE:-$MANTICORE_DATA_DIR/manticore.conf}
+MANTICORE_HTTP_ENDPOINT=${MANTICORE_HTTP_ENDPOINT:-http://127.0.0.1:9308}
+export APP_CODE_DIR APP_DATA_DIR APP_RUNTIME_DIR APP_TMP_DIR APP_BUILD_DIR APP_HOME_DIR AFFINE_HOME ENV_EXPORT_FILE \
+  MANTICORE_DATA_DIR MANTICORE_CONFIG_FILE MANTICORE_HTTP_ENDPOINT

 log() {
  printf '[%s] %s\n' "$(date --iso-8601=seconds)" "$*"
@@ -57,6 +61,33 @@ prepare_data_dirs() {
  chown -R cloudron:cloudron "$APP_DATA_DIR" "$APP_RUNTIME_DIR" "$APP_HOME_DIR"
 }

+prepare_manticore() {
+  log "Preparing Manticore data directory"
+  local buddy_plugins_dir="$MANTICORE_DATA_DIR/plugins/buddy-plugins"
+  mkdir -p "$MANTICORE_DATA_DIR"/{data,binlog,logs,buddy} "$buddy_plugins_dir"
+  cp "$APP_CODE_DIR/manticore/manticore.conf" "$MANTICORE_CONFIG_FILE"
+  local composer_file="$buddy_plugins_dir/composer.json"
+  if [ ! -f "$composer_file" ]; then
+    cat > "$composer_file" <<'JSON'
+{
+  "require": {},
+  "minimum-stability": "dev"
+}
+JSON
+  fi
+  local system_buddy_plugins="/usr/share/manticore/modules/manticore-buddy/buddy-plugins"
+  if [ ! -L "$system_buddy_plugins" ] && [ -w "/usr/share/manticore/modules/manticore-buddy" ]; then
+    rm -rf "$system_buddy_plugins"
+    ln -s "$buddy_plugins_dir" "$system_buddy_plugins"
+  elif [ ! -L "$system_buddy_plugins" ]; then
+    log "Buddy modules directory is read-only; skipping symlink to ${buddy_plugins_dir}"
+  fi
+  mkdir -p /run/manticore
+  chown -R cloudron:cloudron "$MANTICORE_DATA_DIR" /run/manticore
+  record_env_var MANTICORE_CONFIG_FILE "$MANTICORE_CONFIG_FILE"
+  record_env_var MANTICORE_HTTP_ENDPOINT "$MANTICORE_HTTP_ENDPOINT"
+}
+
 prepare_runtime_build_dir() {
  local source_dir="$APP_BUILD_DIR"
  local runtime_build_dir="$APP_RUNTIME_DIR/affine-build"
@@ -81,6 +112,23 @@ configure_database() {
  log "Configured PostgreSQL endpoint"
 }

+ensure_pgvector_extension() {
+  if [ -z "${DATABASE_URL:-}" ]; then
+    log "DATABASE_URL not set; skipping pgvector extension check"
+    return
+  fi
+  if ! command -v psql >/dev/null 2>&1; then
+    log "psql client unavailable; cannot verify pgvector extension"
+    return
+  fi
+  log "Ensuring pgvector extension exists"
+  if psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c "CREATE EXTENSION IF NOT EXISTS vector;" >/dev/null 2>&1; then
+    log "pgvector extension ready"
+  else
+    log "WARNING: Failed to create pgvector extension automatically. Ensure it exists for AI embeddings."
+  fi
+}
+
 configure_redis() {
  require_env CLOUDRON_REDIS_URL
  local redis_info
@@ -228,11 +276,32 @@ PY
      export AFFINE_SERVER_HTTPS=false
    fi
  fi
-  export AFFINE_INDEXER_ENABLED=${AFFINE_INDEXER_ENABLED:-false}
  record_env_var AFFINE_SERVER_EXTERNAL_URL "${AFFINE_SERVER_EXTERNAL_URL:-}"
  record_env_var AFFINE_SERVER_HOST "${AFFINE_SERVER_HOST:-}"
  record_env_var AFFINE_SERVER_HTTPS "${AFFINE_SERVER_HTTPS:-}"
+}
+
+configure_indexer() {
+  export AFFINE_INDEXER_ENABLED=true
+  export AFFINE_INDEXER_SEARCH_PROVIDER=${AFFINE_INDEXER_SEARCH_PROVIDER:-manticoresearch}
+  export AFFINE_INDEXER_SEARCH_ENDPOINT=${AFFINE_INDEXER_SEARCH_ENDPOINT:-$MANTICORE_HTTP_ENDPOINT}
  record_env_var AFFINE_INDEXER_ENABLED "$AFFINE_INDEXER_ENABLED"
+  record_env_var AFFINE_INDEXER_SEARCH_PROVIDER "$AFFINE_INDEXER_SEARCH_PROVIDER"
+  record_env_var AFFINE_INDEXER_SEARCH_ENDPOINT "$AFFINE_INDEXER_SEARCH_ENDPOINT"
+
+  python3 - <<'PY'
+import json
+import os
+from pathlib import Path
+config_path = Path(os.environ['APP_DATA_DIR']) / 'config' / 'config.json'
+data = json.loads(config_path.read_text())
+indexer = data.setdefault('indexer', {})
+indexer['enabled'] = True
+indexer['provider.type'] = os.environ.get('AFFINE_INDEXER_SEARCH_PROVIDER', 'manticoresearch')
+indexer['provider.endpoint'] = os.environ.get('AFFINE_INDEXER_SEARCH_ENDPOINT', 'http://127.0.0.1:9308')
+config_path.write_text(json.dumps(data, indent=2))
+PY
+  log "Configured indexer endpoint"
 }

 configure_auth() {
@@ -306,11 +375,14 @@ PY
 main() {
  export HOME="$APP_HOME_DIR"
  prepare_data_dirs
+  prepare_manticore
  prepare_runtime_build_dir
  configure_database
+  ensure_pgvector_extension
  configure_redis
  configure_mail
  configure_server_metadata
+  configure_indexer
  update_server_config
  configure_auth
  chown -R cloudron:cloudron "$APP_DATA_DIR" "$APP_HOME_DIR"
--- a/supervisord.conf
+++ b/supervisord.conf
@@ -1,7 +1,12 @@
+[unix_http_server]
+file=/run/supervisor.sock
+chmod=0700
+chown=root:root
+
 [supervisord]
 nodaemon=true
 user=root
-logfile=/dev/null
+logfile=/run/supervisord.log
 pidfile=/run/supervisord.pid

 [program:nginx]
@@ -15,6 +20,25 @@ stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 stopsignal=QUIT

+[program:manticore]
+command=/app/code/run-manticore.sh
+autostart=true
+autorestart=true
+startsecs=5
+priority=12
+user=cloudron
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+stopsignal=TERM
+
+[supervisorctl]
+serverurl=unix:///run/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
 [program:affine]
 command=/app/code/run-affine.sh
 autostart=true