diff --git a/BUILD.md b/BUILD.md index d354611..9a13e23 100644 --- a/BUILD.md +++ b/BUILD.md @@ -13,14 +13,14 @@ cloudron build \ --set-build-service builder.docker.due.ren \ --build-service-token e3265de06b1d0e7bb38400539012a8433a74c2c96a17955e \ --set-repository andreasdueren/affine-cloudron \ - --tag 0.25.3 + --tag 0.25.23 ``` ## Deployment Steps 1. Remove any previous dev install of AFFiNE on the Cloudron (always reinstall from scratch). 2. Install the freshly built image: ```bash - cloudron install --location affine.due.ren --image andreasdueren/affine-cloudron:0.25.3 +cloudron install --location affine.due.ren --image andreasdueren/affine-cloudron:0.25.23 ``` 3. When prompted, confirm the app info and wait for Cloudron to report success (abort after ~30 seconds if installation stalls or errors to avoid hanging sessions). 4. Visit `https://affine.due.ren` (or the chosen location) and sign in using Cloudron SSO. diff --git a/CloudronManifest.json b/CloudronManifest.json index d99a7c9..aea6427 100644 --- a/CloudronManifest.json +++ b/CloudronManifest.json @@ -5,8 +5,8 @@ "description": "Next-gen knowledge base that blends docs, whiteboards, and databases for self-hosted teams.", "website": "https://affine.pro", "contactEmail": "support@affine.pro", - "version": "0.25.3", - "changelog": "Upgrade upstream AFFiNE runtime to v0.25.3 and keep Cloudron OIDC wiring", + "version": "0.25.23", + "changelog": "Stop printing wrapper banner so buddy stdout stays parseable for searchd", "icon": "file://icon.png", "manifestVersion": 2, "minBoxVersion": "7.0.0", diff --git a/Dockerfile b/Dockerfile index 4fdcb32..c1bb9a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,9 +14,21 @@ ENV APP_CODE_DIR=/app/code \ RUN mkdir -p "$APP_CODE_DIR" "$APP_DATA_DIR" "$APP_RUNTIME_DIR" "$APP_TMP_DIR" && \ apt-get update && \ - apt-get install -y --no-install-recommends jq python3 ca-certificates curl openssl libjemalloc2 && \ + apt-get install -y --no-install-recommends jq python3 ca-certificates curl openssl libjemalloc2 postgresql-client && \ rm -rf /var/lib/apt/lists/* +RUN curl -fsSL https://repo.manticoresearch.com/GPG-KEY-manticore > /tmp/manticore.key && \ + curl -fsSL https://repo.manticoresearch.com/GPG-KEY-SHA256-manticore >> /tmp/manticore.key && \ + gpg --dearmor -o /usr/share/keyrings/manticore.gpg /tmp/manticore.key && \ + rm /tmp/manticore.key && \ + echo "deb [signed-by=/usr/share/keyrings/manticore.gpg] https://repo.manticoresearch.com/repository/manticoresearch_jammy/ jammy main" > /etc/apt/sources.list.d/manticore.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends manticore manticore-extra && \ + rm -rf /var/lib/apt/lists/* + +RUN ln -sf /usr/share/manticore/modules/manticore-buddy/bin/manticore-buddy /usr/bin/manticore-buddy +RUN chown -R cloudron:cloudron /usr/share/manticore + # bring in the upstream runtime and packaged server artifacts COPY --from=upstream /usr/local /usr/local COPY --from=upstream /opt /opt @@ -25,13 +37,16 @@ COPY --from=upstream /app "$APP_BUILD_DIR" # configuration, launch scripts, and defaults COPY start.sh "$APP_CODE_DIR/start.sh" COPY run-affine.sh "$APP_CODE_DIR/run-affine.sh" +COPY run-manticore.sh "$APP_CODE_DIR/run-manticore.sh" +COPY run-buddy.sh "$APP_CODE_DIR/run-buddy.sh" COPY nginx.conf "$APP_CODE_DIR/nginx.conf" COPY supervisord.conf "$APP_CODE_DIR/supervisord.conf" COPY config.example.json "$APP_CODE_DIR/config.example.json" COPY tmp_data/ "$APP_TMP_DIR/" +COPY manticore/ "$APP_CODE_DIR/manticore/" -RUN chmod +x "$APP_CODE_DIR/start.sh" "$APP_CODE_DIR/run-affine.sh" && \ - chown cloudron:cloudron "$APP_CODE_DIR/start.sh" "$APP_CODE_DIR/run-affine.sh" && \ +RUN chmod +x "$APP_CODE_DIR/start.sh" "$APP_CODE_DIR/run-affine.sh" "$APP_CODE_DIR/run-manticore.sh" "$APP_CODE_DIR/run-buddy.sh" && \ + chown cloudron:cloudron "$APP_CODE_DIR/start.sh" "$APP_CODE_DIR/run-affine.sh" "$APP_CODE_DIR/run-manticore.sh" "$APP_CODE_DIR/run-buddy.sh" && \ chown -R cloudron:cloudron "$APP_DATA_DIR" "$APP_RUNTIME_DIR" "$APP_TMP_DIR" EXPOSE 3000 diff --git a/manticore/manticore.conf b/manticore/manticore.conf new file mode 100644 index 0000000..a97afe7 --- /dev/null +++ b/manticore/manticore.conf @@ -0,0 +1,23 @@ +# +# Manticore Search configuration for AFFiNE on Cloudron. +# Keeps all runtime state under /app/data so backups include the indexer data. +# + +common { + plugin_dir = /app/data/manticore/plugins +} + +searchd { + listen = 127.0.0.1:9306:mysql41 + listen = 127.0.0.1:9308:http + listen = 127.0.0.1:9312 + listen = /run/manticore/manticore.sock:mysql41 + + log = /app/data/manticore/logs/searchd.log + query_log = /app/data/manticore/logs/query.log + binlog_path = /app/data/manticore/binlog + data_dir = /app/data/manticore/data + pid_file = /run/manticore/searchd.pid + mysql_version_string = 8.0.33 + buddy_path = /app/code/run-buddy.sh --disable-telemetry +} diff --git a/run-affine.sh b/run-affine.sh index 230b74a..ec26836 100644 --- a/run-affine.sh +++ b/run-affine.sh @@ -89,7 +89,9 @@ PY else export AFFINE_SERVER_HTTPS=false fi - export AFFINE_INDEXER_ENABLED="${AFFINE_INDEXER_ENABLED:-false}" + export AFFINE_INDEXER_ENABLED="${AFFINE_INDEXER_ENABLED:-true}" + export AFFINE_INDEXER_SEARCH_PROVIDER="${AFFINE_INDEXER_SEARCH_PROVIDER:-manticoresearch}" + export AFFINE_INDEXER_SEARCH_ENDPOINT="${AFFINE_INDEXER_SEARCH_ENDPOINT:-http://127.0.0.1:9308}" } ensure_runtime_envs() { @@ -99,6 +101,45 @@ ensure_runtime_envs() { ensure_server_env } +# Helper to parse indexer endpoint into host/port for readiness checks +wait_for_indexer() { + if [ "${AFFINE_INDEXER_ENABLED:-false}" != "true" ]; then + return + fi + local endpoint="${AFFINE_INDEXER_SEARCH_ENDPOINT:-}" + if [ -z "$endpoint" ]; then + return + fi + log "Waiting for indexer endpoint ${endpoint}" + if python3 - "$endpoint" <<'PY'; then +import socket +import sys +import time +from urllib.parse import urlparse + +endpoint = sys.argv[1] +if not endpoint.startswith(('http://', 'https://')): + endpoint = 'http://' + endpoint +parsed = urlparse(endpoint) +host = parsed.hostname +port = parsed.port or (443 if parsed.scheme == 'https' else 80) +if not host or not port: + sys.exit(1) + +for _ in range(60): + try: + with socket.create_connection((host, port), timeout=2): + sys.exit(0) + except OSError: + time.sleep(1) +sys.exit(1) +PY + log "Indexer is ready" + else + log "Indexer at ${endpoint} not reachable after waiting, continuing startup" + fi +} + patch_upload_limits() { local target="$APP_DIR/dist/main.js" if [ ! -f "$target" ]; then @@ -228,6 +269,7 @@ NODE log "Running AFFiNE pre-deployment migrations" ensure_runtime_envs +wait_for_indexer node ./scripts/self-host-predeploy.js patch_upload_limits grant_team_plan_features diff --git a/run-buddy.sh b/run-buddy.sh new file mode 100644 index 0000000..9794206 --- /dev/null +++ b/run-buddy.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail + +ENV_EXPORT_FILE=${ENV_EXPORT_FILE:-/run/affine/runtime.env} + +if [ -f "$ENV_EXPORT_FILE" ]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_EXPORT_FILE" + set +a +fi + +exec /usr/bin/manticore-buddy "$@" diff --git a/run-manticore.sh b/run-manticore.sh new file mode 100644 index 0000000..3d9158e --- /dev/null +++ b/run-manticore.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -euo pipefail + +MANTICORE_CONF=${MANTICORE_CONF:-/app/data/manticore/manticore.conf} +MANTICORE_RUN_DIR=${MANTICORE_RUN_DIR:-/run/manticore} + +mkdir -p "$MANTICORE_RUN_DIR" +rm -f "$MANTICORE_RUN_DIR/searchd.pid" + +exec /usr/bin/searchd --nodetach -c "$MANTICORE_CONF" diff --git a/start.sh b/start.sh index 506a7ff..15a8cf9 100644 --- a/start.sh +++ b/start.sh @@ -9,7 +9,11 @@ APP_BUILD_DIR=${APP_BUILD_DIR:-/app/code/affine} APP_HOME_DIR=${APP_HOME_DIR:-/app/data/home} AFFINE_HOME=${AFFINE_HOME:-$APP_HOME_DIR/.affine} ENV_EXPORT_FILE=${ENV_EXPORT_FILE:-$APP_RUNTIME_DIR/runtime.env} -export APP_CODE_DIR APP_DATA_DIR APP_RUNTIME_DIR APP_TMP_DIR APP_BUILD_DIR APP_HOME_DIR AFFINE_HOME ENV_EXPORT_FILE +MANTICORE_DATA_DIR=${MANTICORE_DATA_DIR:-$APP_DATA_DIR/manticore} +MANTICORE_CONFIG_FILE=${MANTICORE_CONFIG_FILE:-$MANTICORE_DATA_DIR/manticore.conf} +MANTICORE_HTTP_ENDPOINT=${MANTICORE_HTTP_ENDPOINT:-http://127.0.0.1:9308} +export APP_CODE_DIR APP_DATA_DIR APP_RUNTIME_DIR APP_TMP_DIR APP_BUILD_DIR APP_HOME_DIR AFFINE_HOME ENV_EXPORT_FILE \ + MANTICORE_DATA_DIR MANTICORE_CONFIG_FILE MANTICORE_HTTP_ENDPOINT log() { printf '[%s] %s\n' "$(date --iso-8601=seconds)" "$*" @@ -57,6 +61,33 @@ prepare_data_dirs() { chown -R cloudron:cloudron "$APP_DATA_DIR" "$APP_RUNTIME_DIR" "$APP_HOME_DIR" } +prepare_manticore() { + log "Preparing Manticore data directory" + local buddy_plugins_dir="$MANTICORE_DATA_DIR/plugins/buddy-plugins" + mkdir -p "$MANTICORE_DATA_DIR"/{data,binlog,logs,buddy} "$buddy_plugins_dir" + cp "$APP_CODE_DIR/manticore/manticore.conf" "$MANTICORE_CONFIG_FILE" + local composer_file="$buddy_plugins_dir/composer.json" + if [ ! -f "$composer_file" ]; then + cat > "$composer_file" <<'JSON' +{ + "require": {}, + "minimum-stability": "dev" +} +JSON + fi + local system_buddy_plugins="/usr/share/manticore/modules/manticore-buddy/buddy-plugins" + if [ ! -L "$system_buddy_plugins" ] && [ -w "/usr/share/manticore/modules/manticore-buddy" ]; then + rm -rf "$system_buddy_plugins" + ln -s "$buddy_plugins_dir" "$system_buddy_plugins" + elif [ ! -L "$system_buddy_plugins" ]; then + log "Buddy modules directory is read-only; skipping symlink to ${buddy_plugins_dir}" + fi + mkdir -p /run/manticore + chown -R cloudron:cloudron "$MANTICORE_DATA_DIR" /run/manticore + record_env_var MANTICORE_CONFIG_FILE "$MANTICORE_CONFIG_FILE" + record_env_var MANTICORE_HTTP_ENDPOINT "$MANTICORE_HTTP_ENDPOINT" +} + prepare_runtime_build_dir() { local source_dir="$APP_BUILD_DIR" local runtime_build_dir="$APP_RUNTIME_DIR/affine-build" @@ -81,6 +112,23 @@ configure_database() { log "Configured PostgreSQL endpoint" } +ensure_pgvector_extension() { + if [ -z "${DATABASE_URL:-}" ]; then + log "DATABASE_URL not set; skipping pgvector extension check" + return + fi + if ! command -v psql >/dev/null 2>&1; then + log "psql client unavailable; cannot verify pgvector extension" + return + fi + log "Ensuring pgvector extension exists" + if psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c "CREATE EXTENSION IF NOT EXISTS vector;" >/dev/null 2>&1; then + log "pgvector extension ready" + else + log "WARNING: Failed to create pgvector extension automatically. Ensure it exists for AI embeddings." + fi +} + configure_redis() { require_env CLOUDRON_REDIS_URL local redis_info @@ -228,11 +276,32 @@ PY export AFFINE_SERVER_HTTPS=false fi fi - export AFFINE_INDEXER_ENABLED=${AFFINE_INDEXER_ENABLED:-false} record_env_var AFFINE_SERVER_EXTERNAL_URL "${AFFINE_SERVER_EXTERNAL_URL:-}" record_env_var AFFINE_SERVER_HOST "${AFFINE_SERVER_HOST:-}" record_env_var AFFINE_SERVER_HTTPS "${AFFINE_SERVER_HTTPS:-}" +} + +configure_indexer() { + export AFFINE_INDEXER_ENABLED=true + export AFFINE_INDEXER_SEARCH_PROVIDER=${AFFINE_INDEXER_SEARCH_PROVIDER:-manticoresearch} + export AFFINE_INDEXER_SEARCH_ENDPOINT=${AFFINE_INDEXER_SEARCH_ENDPOINT:-$MANTICORE_HTTP_ENDPOINT} record_env_var AFFINE_INDEXER_ENABLED "$AFFINE_INDEXER_ENABLED" + record_env_var AFFINE_INDEXER_SEARCH_PROVIDER "$AFFINE_INDEXER_SEARCH_PROVIDER" + record_env_var AFFINE_INDEXER_SEARCH_ENDPOINT "$AFFINE_INDEXER_SEARCH_ENDPOINT" + + python3 - <<'PY' +import json +import os +from pathlib import Path +config_path = Path(os.environ['APP_DATA_DIR']) / 'config' / 'config.json' +data = json.loads(config_path.read_text()) +indexer = data.setdefault('indexer', {}) +indexer['enabled'] = True +indexer['provider.type'] = os.environ.get('AFFINE_INDEXER_SEARCH_PROVIDER', 'manticoresearch') +indexer['provider.endpoint'] = os.environ.get('AFFINE_INDEXER_SEARCH_ENDPOINT', 'http://127.0.0.1:9308') +config_path.write_text(json.dumps(data, indent=2)) +PY + log "Configured indexer endpoint" } configure_auth() { @@ -306,11 +375,14 @@ PY main() { export HOME="$APP_HOME_DIR" prepare_data_dirs + prepare_manticore prepare_runtime_build_dir configure_database + ensure_pgvector_extension configure_redis configure_mail configure_server_metadata + configure_indexer update_server_config configure_auth chown -R cloudron:cloudron "$APP_DATA_DIR" "$APP_HOME_DIR" diff --git a/supervisord.conf b/supervisord.conf index 2b0d9ad..e4ac7cd 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -1,7 +1,12 @@ +[unix_http_server] +file=/run/supervisor.sock +chmod=0700 +chown=root:root + [supervisord] nodaemon=true user=root -logfile=/dev/null +logfile=/run/supervisord.log pidfile=/run/supervisord.pid [program:nginx] @@ -15,6 +20,25 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 stopsignal=QUIT +[program:manticore] +command=/app/code/run-manticore.sh +autostart=true +autorestart=true +startsecs=5 +priority=12 +user=cloudron +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +stopsignal=TERM + +[supervisorctl] +serverurl=unix:///run/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + [program:affine] command=/app/code/run-affine.sh autostart=true