Support optional S3 replication

2025-11-17 13:18:12 -06:00
parent 8f3a34a277
commit 8a05a3627d
4 changed files with 135 additions and 29 deletions
--- a/BUILD-INSTRUCTIONS.md
+++ b/BUILD-INSTRUCTIONS.md
@@ -20,7 +20,7 @@ cloudron install \
 ```

 ## After Install
-1. **S3** – In Cloudron File Manager open `/app/data/config/s3.env`, fill in your endpoint/region/bucket/access/secret, then restart the app from the dashboard.
+1. **S3** – In Cloudron File Manager open `/app/data/config/s3.env`, fill in your endpoint/region/bucket/access/secret, then restart the app from the dashboard. Optional replication: add `S3_SECONDARY_ENDPOINT`, `S3_SECONDARY_ACCESS_KEY`, `S3_SECONDARY_SECRET_KEY`, and `S3_SECONDARY_DC` (plus overrides for region/bucket/prefix) to mirror uploads to a second provider. See Ente’s [object storage guide](https://ente.io/help/self-hosting/administration/object-storage).
 2. **Secondary hostnames** – During installation Cloudron now prompts for hostnames for the Accounts/Auth/Cast/Albums/Family web apps (powered by `httpPorts`). Ensure matching DNS records exist that point to the primary app domain. If you use Cloudron-managed DNS, those records are created automatically; otherwise create CNAME/A records such as `accounts.<app-domain> → <app-domain>`.

 Once DNS propagates, use the dedicated hosts (defaults shown below — substitute the names you selected during install):
--- a/POSTINSTALL.md
+++ b/POSTINSTALL.md
@@ -53,6 +53,7 @@ Supported variables:
 - `S3_ACCESS_KEY`
 - `S3_SECRET_KEY`
 - `S3_PREFIX` (optional path prefix)
+- Optional replication: define `S3_SECONDARY_ENDPOINT`, `S3_SECONDARY_ACCESS_KEY`, `S3_SECONDARY_SECRET_KEY`, and `S3_SECONDARY_DC` (plus optional overrides for region/bucket/prefix) to mirror uploads to a second object store. See [Ente’s object storage guide](https://ente.io/help/self-hosting/administration/object-storage) for sample setups.

 ## Required: Secondary Hostnames

--- a/README.md
+++ b/README.md
@@ -55,7 +55,7 @@ The app is configured automatically using Cloudron's environment variables for:

 After installing on Cloudron remember to:

-1. Open the File Manager for the app, edit `/app/data/config/s3.env` with your object storage endpoint/keys, and restart the app. If you are using Cloudflare R2 or another S3-compatible service, configure the bucket’s CORS policy to allow the Ente frontends (e.g. `https://ente.due.ren`, `https://accounts.due.ren`, `https://cast.due.ren`, etc.) so that cast/slideshow playback can fetch signed URLs directly from storage.
+1. Open the File Manager for the app, edit `/app/data/config/s3.env` with your object storage endpoint/keys, and restart the app. If you are using Cloudflare R2 or another S3-compatible service, configure the bucket’s CORS policy to allow the Ente frontends (e.g. `https://ente.due.ren`, `https://accounts.due.ren`, `https://cast.due.ren`, etc.) so that cast/slideshow playback can fetch signed URLs directly from storage. To enable replication, add `S3_SECONDARY_ENDPOINT`, `S3_SECONDARY_ACCESS_KEY`, `S3_SECONDARY_SECRET_KEY`, and `S3_SECONDARY_DC` (plus any overrides for region/bucket/prefix) as described in the [object storage guide](https://ente.io/help/self-hosting/administration/object-storage).
 2. When prompted during installation, pick hostnames for the Accounts/Auth/Cast/Albums/Family web apps (they are exposed via Cloudron `httpPorts`). Ensure matching DNS records exist; Cloudron-managed DNS creates them automatically, otherwise point CNAME/A records such as `accounts.<app-domain>` at the primary hostname.
 3. To persist tweaks to Museum (for example, seeding super-admin or whitelist entries), create `/app/data/config/museum.override.yaml`. Its contents are appended to the generated `museum/configurations/local.yaml` on every start, so you only need to declare the keys you want to override.
   ```yaml
--- a/start.sh
+++ b/start.sh
@@ -177,6 +177,14 @@ if [ ! -f "$S3_CONFIG_FILE" ]; then
 #   S3_ACCESS_KEY=your-access-key
 #   S3_SECRET_KEY=your-secret-key
 #   S3_PREFIX=optional/path/prefix
+# Optional replication settings (secondary object storage):
+#   S3_SECONDARY_ENDPOINT=https://secondary.s3-provider.com
+#   S3_SECONDARY_REGION=us-west-1
+#   S3_SECONDARY_BUCKET=ente-data-backup
+#   S3_SECONDARY_ACCESS_KEY=secondary-access-key
+#   S3_SECONDARY_SECRET_KEY=secondary-secret-key
+#   S3_SECONDARY_PREFIX=optional/path/prefix
+#   S3_SECONDARY_DC=b2-us-west
 #
 # Example for Cloudflare R2 (replace placeholders):
 #S3_ENDPOINT=https://<account-id>.r2.cloudflarestorage.com
@@ -186,8 +194,14 @@ if [ ! -f "$S3_CONFIG_FILE" ]; then
 #S3_SECRET_KEY=R2_SECRET_KEY
 #S3_FORCE_PATH_STYLE=true
 #S3_PRIMARY_DC=b2-eu-cen
-#S3_SECONDARY_DC=b2-eu-cen
 #S3_DERIVED_DC=b2-eu-cen
+# Optional replication to Backblaze B2:
+#S3_SECONDARY_ENDPOINT=https://s3.us-west-002.backblazeb2.com
+#S3_SECONDARY_REGION=us-west-002
+#S3_SECONDARY_BUCKET=ente
+#S3_SECONDARY_ACCESS_KEY=B2_ACCESS_KEY
+#S3_SECONDARY_SECRET_KEY=B2_SECRET_KEY
+#S3_SECONDARY_DC=b2-us-west
 #
 # Example for Backblaze B2 (replace placeholders):
 #S3_ENDPOINT=https://s3.us-west-002.backblazeb2.com
@@ -197,8 +211,14 @@ if [ ! -f "$S3_CONFIG_FILE" ]; then
 #S3_SECRET_KEY=B2_SECRET_KEY
 #S3_FORCE_PATH_STYLE=true
 #S3_PRIMARY_DC=b2-eu-cen
-#S3_SECONDARY_DC=b2-eu-cen
 #S3_DERIVED_DC=b2-eu-cen
+# Optional replication to Cloudflare R2:
+#S3_SECONDARY_ENDPOINT=https://<account-id>.r2.cloudflarestorage.com
+#S3_SECONDARY_REGION=auto
+#S3_SECONDARY_BUCKET=ente-replica
+#S3_SECONDARY_ACCESS_KEY=R2_ACCESS_KEY
+#S3_SECONDARY_SECRET_KEY=R2_SECRET_KEY
+#S3_SECONDARY_DC=r2-eu
 EOF_S3
  chown cloudron:cloudron "$S3_CONFIG_FILE"
  chmod 600 "$S3_CONFIG_FILE"
@@ -212,6 +232,25 @@ if [ -f "$S3_CONFIG_FILE" ]; then
 fi
 set -u

+parse_s3_endpoint() {
+  local raw="$1"
+  local prefix="$2"
+  local host_var="$3"
+  local prefix_var="$4"
+  local host="${raw#https://}"
+  host="${host#http://}"
+  host="${host%%/}"
+  local path="${host#*/}"
+  if [ "$path" != "$host" ]; then
+    if [ -z "$prefix" ]; then
+      prefix="$path"
+    fi
+    host="${host%%/*}"
+  fi
+  printf -v "$host_var" "%s" "$host"
+  printf -v "$prefix_var" "%s" "$prefix"
+}
+
 S3_ENDPOINT="${S3_ENDPOINT:-${ENTE_S3_ENDPOINT:-}}"
 S3_REGION="${S3_REGION:-${ENTE_S3_REGION:-}}"
 S3_BUCKET="${S3_BUCKET:-${ENTE_S3_BUCKET:-}}"
@@ -219,6 +258,16 @@ S3_ACCESS_KEY="${S3_ACCESS_KEY:-${ENTE_S3_ACCESS_KEY:-}}"
 S3_SECRET_KEY="${S3_SECRET_KEY:-${ENTE_S3_SECRET_KEY:-}}"
 S3_PREFIX="${S3_PREFIX:-${ENTE_S3_PREFIX:-}}"

+S3_SECONDARY_ENDPOINT="${S3_SECONDARY_ENDPOINT:-${ENTE_S3_SECONDARY_ENDPOINT:-}}"
+S3_SECONDARY_REGION="${S3_SECONDARY_REGION:-${ENTE_S3_SECONDARY_REGION:-}}"
+S3_SECONDARY_BUCKET="${S3_SECONDARY_BUCKET:-${ENTE_S3_SECONDARY_BUCKET:-}}"
+S3_SECONDARY_ACCESS_KEY="${S3_SECONDARY_ACCESS_KEY:-${ENTE_S3_SECONDARY_ACCESS_KEY:-}}"
+S3_SECONDARY_SECRET_KEY="${S3_SECONDARY_SECRET_KEY:-${ENTE_S3_SECONDARY_SECRET_KEY:-}}"
+S3_SECONDARY_PREFIX="${S3_SECONDARY_PREFIX:-${ENTE_S3_SECONDARY_PREFIX:-}}"
+S3_SECONDARY_DC_RAW="${ENTE_S3_SECONDARY_DC:-}"
+S3_SECONDARY_ENABLED=false
+S3_SECONDARY_ENDPOINT_HOST=""
+
 if [ -z "$S3_ENDPOINT" ] || [ -z "$S3_REGION" ] || [ -z "$S3_BUCKET" ] || [ -z "$S3_ACCESS_KEY" ] || [ -z "$S3_SECRET_KEY" ]; then
  log ERROR "Missing S3 configuration. Update $S3_CONFIG_FILE or set environment variables."
  log ERROR "The application will start in configuration mode. Please configure S3 and restart."
@@ -228,18 +277,12 @@ else
 fi

 if [ "$S3_NOT_CONFIGURED" = "false" ]; then
-  S3_ENDPOINT_HOST="${S3_ENDPOINT#https://}"
-  S3_ENDPOINT_HOST="${S3_ENDPOINT_HOST#http://}"
-  S3_ENDPOINT_HOST="${S3_ENDPOINT_HOST%%/}"
-  S3_ENDPOINT_PATH="${S3_ENDPOINT_HOST#*/}"
-  if [ "$S3_ENDPOINT_PATH" != "$S3_ENDPOINT_HOST" ]; then
-    if [ -z "$S3_PREFIX" ]; then
-      S3_PREFIX="$S3_ENDPOINT_PATH"
-    fi
-    S3_ENDPOINT_HOST="${S3_ENDPOINT_HOST%%/*}"
-  fi
-
+  parse_s3_endpoint "$S3_ENDPOINT" "$S3_PREFIX" S3_ENDPOINT_HOST S3_PREFIX
  log INFO "Using S3 endpoint $S3_ENDPOINT_HOST (region $S3_REGION, bucket $S3_BUCKET)"
+  if [ "$S3_SECONDARY_ENABLED" = true ]; then
+    parse_s3_endpoint "$S3_SECONDARY_ENDPOINT" "$S3_SECONDARY_PREFIX" S3_SECONDARY_ENDPOINT_HOST S3_SECONDARY_PREFIX
+    log INFO "Secondary S3 replication endpoint $S3_SECONDARY_ENDPOINT_HOST (region $S3_SECONDARY_REGION, bucket $S3_SECONDARY_BUCKET)"
+  fi
  S3_REGION_LOWER="$(printf '%s' "$S3_REGION" | tr '[:upper:]' '[:lower:]')"
  if printf '%s' "$S3_ENDPOINT_HOST" | grep -q '\.r2\.cloudflarestorage\.com$' && [ "$S3_REGION_LOWER" != "auto" ]; then
    log WARN "Cloudflare R2 endpoints require S3_REGION=auto; current value '$S3_REGION' may cause upload failures"
@@ -247,6 +290,11 @@ if [ "$S3_NOT_CONFIGURED" = "false" ]; then
 else
  S3_ENDPOINT_HOST="s3.example.com"
  log WARN "S3 not configured - using placeholder values"
+  if [ "$S3_SECONDARY_ENABLED" = true ]; then
+    log WARN "Disabling secondary S3 configuration because the primary endpoint is not configured"
+    S3_SECONDARY_ENABLED=false
+    S3_SECONDARY_DC=""
+  fi
 fi

 DEFAULT_FORCE_PATH_STYLE="true"
@@ -261,9 +309,41 @@ S3_FORCE_PATH_STYLE="$(printf '%s' "$S3_FORCE_PATH_STYLE_RAW" | tr '[:upper:]' '
 S3_ARE_LOCAL_BUCKETS="$(printf '%s' "${S3_ARE_LOCAL_BUCKETS:-${ENTE_S3_ARE_LOCAL_BUCKETS:-false}}" | tr '[:upper:]' '[:lower:]')"

 S3_PRIMARY_DC="${ENTE_S3_PRIMARY_DC:-b2-eu-cen}"
-S3_SECONDARY_DC="${ENTE_S3_SECONDARY_DC:-$S3_PRIMARY_DC}"
 S3_DERIVED_DC="${ENTE_S3_DERIVED_DC:-$S3_PRIMARY_DC}"

+S3_SECONDARY_ENV_PRESENT=false
+for value in "$S3_SECONDARY_ENDPOINT" "$S3_SECONDARY_REGION" "$S3_SECONDARY_BUCKET" "$S3_SECONDARY_ACCESS_KEY" "$S3_SECONDARY_SECRET_KEY" "$S3_SECONDARY_PREFIX" "$S3_SECONDARY_DC_RAW"; do
+  if [ -n "$value" ]; then
+    S3_SECONDARY_ENV_PRESENT=true
+    break
+  fi
+done
+
+if [ "$S3_NOT_CONFIGURED" = "false" ] && [ "$S3_SECONDARY_ENV_PRESENT" = true ]; then
+  S3_SECONDARY_REGION="${S3_SECONDARY_REGION:-$S3_REGION}"
+  S3_SECONDARY_BUCKET="${S3_SECONDARY_BUCKET:-$S3_BUCKET}"
+  S3_SECONDARY_PREFIX="${S3_SECONDARY_PREFIX:-$S3_PREFIX}"
+  MISSING_SECONDARY_VARS=()
+  [ -z "$S3_SECONDARY_ENDPOINT" ] && MISSING_SECONDARY_VARS+=("S3_SECONDARY_ENDPOINT")
+  [ -z "$S3_SECONDARY_ACCESS_KEY" ] && MISSING_SECONDARY_VARS+=("S3_SECONDARY_ACCESS_KEY")
+  [ -z "$S3_SECONDARY_SECRET_KEY" ] && MISSING_SECONDARY_VARS+=("S3_SECONDARY_SECRET_KEY")
+  if [ "${#MISSING_SECONDARY_VARS[@]}" -gt 0 ]; then
+    log ERROR "Secondary S3 configuration incomplete (missing: ${MISSING_SECONDARY_VARS[*]}). Replication disabled."
+    S3_SECONDARY_ENABLED=false
+    S3_SECONDARY_DC=""
+  else
+    S3_SECONDARY_ENABLED=true
+    if [ -n "$S3_SECONDARY_DC_RAW" ]; then
+      S3_SECONDARY_DC="$S3_SECONDARY_DC_RAW"
+    else
+      S3_SECONDARY_DC="${S3_PRIMARY_DC}-secondary"
+    fi
+  fi
+else
+  S3_SECONDARY_ENABLED=false
+  S3_SECONDARY_DC=""
+fi
+
 S3_DCS=()
 add_s3_dc() {
  local candidate="$1"
@@ -279,11 +359,39 @@ add_s3_dc() {
 }

 add_s3_dc "$S3_PRIMARY_DC"
-add_s3_dc "$S3_SECONDARY_DC"
+if [ "$S3_SECONDARY_ENABLED" = true ]; then
+  add_s3_dc "$S3_SECONDARY_DC"
+fi
 add_s3_dc "$S3_DERIVED_DC"

+write_s3_dc_block() {
+  local dc="$1"
+  local key="$2"
+  local secret="$3"
+  local endpoint="$4"
+  local region="$5"
+  local bucket="$6"
+  local prefix="$7"
+  cat >> "$MUSEUM_CONFIG" <<EOF_CFG
+  $dc:
+    key: "$key"
+    secret: "$secret"
+    endpoint: "$endpoint"
+    region: "$region"
+    bucket: "$bucket"
+EOF_CFG
+  if [ -n "$prefix" ]; then
+    printf '    path_prefix: "%s"\n' "$prefix" >> "$MUSEUM_CONFIG"
+  fi
+  printf '\n' >> "$MUSEUM_CONFIG"
+}
+
 S3_PREFIX_DISPLAY="${S3_PREFIX:-<none>}"
 log INFO "Resolved S3 configuration: host=$S3_ENDPOINT_HOST region=$S3_REGION pathStyle=$S3_FORCE_PATH_STYLE localBuckets=$S3_ARE_LOCAL_BUCKETS primaryDC=$S3_PRIMARY_DC derivedDC=$S3_DERIVED_DC prefix=$S3_PREFIX_DISPLAY"
+if [ "$S3_SECONDARY_ENABLED" = true ]; then
+  S3_SECONDARY_PREFIX_DISPLAY="${S3_SECONDARY_PREFIX:-<none>}"
+  log INFO "Secondary replication target: host=$S3_SECONDARY_ENDPOINT_HOST region=$S3_SECONDARY_REGION dc=$S3_SECONDARY_DC prefix=$S3_SECONDARY_PREFIX_DISPLAY"
+fi

 DEFAULT_GIN_TRUSTED_PROXIES="127.0.0.1,::1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
 GIN_TRUSTED_PROXIES="${GIN_TRUSTED_PROXIES:-$DEFAULT_GIN_TRUSTED_PROXIES}"
@@ -412,6 +520,10 @@ fi

 # Always regenerate Museum config to pick up S3 changes
 log INFO "Rendering Museum configuration"
+HOT_STORAGE_SECONDARY_LINE=""
+if [ "$S3_SECONDARY_ENABLED" = true ]; then
+  HOT_STORAGE_SECONDARY_LINE="    secondary: ${S3_SECONDARY_DC}"
+fi
 cat > "$MUSEUM_CONFIG" <<EOF_CFG
 log-file: ""
 http:
@@ -440,23 +552,16 @@ s3:
  use_path_style_urls: ${S3_FORCE_PATH_STYLE}
  hot_storage:
    primary: ${S3_PRIMARY_DC}
-    secondary: ${S3_SECONDARY_DC}
+${HOT_STORAGE_SECONDARY_LINE}
  derived-storage: ${S3_DERIVED_DC}
 EOF_CFG

 for dc in "${S3_DCS[@]}"; do
-  cat >> "$MUSEUM_CONFIG" <<EOF_CFG
-  $dc:
-    key: "$S3_ACCESS_KEY"
-    secret: "$S3_SECRET_KEY"
-    endpoint: "$S3_ENDPOINT_HOST"
-    region: "$S3_REGION"
-    bucket: "$S3_BUCKET"
-EOF_CFG
-  if [ -n "$S3_PREFIX" ]; then
-    printf '    path_prefix: "%s"\n' "$S3_PREFIX" >> "$MUSEUM_CONFIG"
+  if [ "$S3_SECONDARY_ENABLED" = true ] && [ "$dc" = "$S3_SECONDARY_DC" ]; then
+    write_s3_dc_block "$dc" "$S3_SECONDARY_ACCESS_KEY" "$S3_SECONDARY_SECRET_KEY" "$S3_SECONDARY_ENDPOINT_HOST" "$S3_SECONDARY_REGION" "$S3_SECONDARY_BUCKET" "$S3_SECONDARY_PREFIX"
+    continue
  fi
-  printf '\n' >> "$MUSEUM_CONFIG"
+  write_s3_dc_block "$dc" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$S3_ENDPOINT_HOST" "$S3_REGION" "$S3_BUCKET" "$S3_PREFIX"
 done

 cat >> "$MUSEUM_CONFIG" <<EOF_CFG