feat: 新增 S3 兼容云存储同步 (#1351)

* Add S3 Cloud Sync design document Design for adding AWS S3 as a new Cloud Sync backend alongside WebDAV. Hybrid approach: extract shared sync protocol, add independent S3 transport. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Add S3 cloud sync implementation design (reqwest + Sig V4) Updated design based on 2026-03-06 draft: switches from rust-s3 crate to hand-rolled AWS Sig V4 on existing reqwest for broader S3-compatible service support (AWS, MinIO, R2, Alibaba OSS, Tencent COS, Huawei OBS). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Add S3 cloud sync implementation plan (11 tasks, TDD) Detailed step-by-step plan covering: sync_protocol extraction, S3 Sig V4 transport, settings, sync/auto-sync modules, Tauri commands, frontend presets/dynamic form, and i18n. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * deps: add hmac crate for S3 Sig V4 signing Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * refactor: extract sync_protocol.rs from webdav_sync.rs for shared use Move transport-agnostic sync protocol logic (constants, types, snapshot building, manifest validation, artifact verification, snapshot application, utilities) into a new shared sync_protocol module so both WebDAV and the upcoming S3 transport can reuse it. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: use transport-neutral error keys in sync_protocol Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add S3 transport layer with AWS Sig V4 signing Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add S3SyncSettings to AppSettings Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add S3 sync module with upload/download/fetch Implements the S3 sync protocol layer (s3_sync.rs) that combines the shared sync_protocol with the S3 transport. Mirrors the WebDAV sync module structure with independent sync mutex, connection check, upload, download, fetch_remote_info, and sync status persistence. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add S3 auto sync worker with debounce Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add S3 sync Tauri commands and auto sync worker startup Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add S3 sync TypeScript types and API layer Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add S3 sync i18n translations (en/zh/ja) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add S3 sync presets and dynamic form to sync settings Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: preserve HTTP scheme for S3 custom endpoints (MinIO support) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * test: add live S3 integration tests (env-var driven, --ignored) Run with: S3_TEST_AK=... S3_TEST_SK=... S3_TEST_BUCKET=... cargo test --lib services::s3::integration_tests -- --ignored Verifies test_connection, put_object, get_object, head_object, and 404 handling against a real S3 bucket using the project's own Sig V4 signing. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * chore: remove internal design docs before PR * fix: wire S3 auto-sync to DB hook & sync UI state on async load - P1: Add s3_auto_sync::notify_db_changed call in SQLite update_hook so S3 auto-sync worker receives DB change signals (was only wired for WebDAV, leaving S3 worker idle) - P2: Add useEffect to update syncType selector when s3Config loads asynchronously, preventing stale "webdav" default for S3 users * fix: satisfy clippy for s3 sync * fix: address s3 sync review feedback --------- Co-authored-by: Keith (via OpenClaw) <keithyt06@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Jason <farion1231@gmail.com>
2026-08-02 10:21:16 +08:00 · 2026-06-04 22:18:51 +08:00
parent ea95f39adf
commit 2a24da517f
26 changed files with 4393 additions and 795 deletions
@@ -0,0 +1,270 @@
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::sync::OnceLock;
+use std::time::{Duration, Instant};
+
+use serde_json::json;
+use tauri::{AppHandle, Emitter};
+use tokio::sync::mpsc::error::TrySendError;
+use tokio::sync::mpsc::{channel, Receiver, Sender};
+
+use crate::error::AppError;
+use crate::services::s3_sync;
+use crate::settings::{self, S3SyncSettings};
+
+const AUTO_SYNC_DEBOUNCE_MS: u64 = 1000;
+pub(crate) const MAX_AUTO_SYNC_WAIT_MS: u64 = 10_000;
+
+static DB_CHANGE_TX: OnceLock<Sender<String>> = OnceLock::new();
+static AUTO_SYNC_SUPPRESS_DEPTH: AtomicUsize = AtomicUsize::new(0);
+
+pub(crate) struct AutoSyncSuppressionGuard;
+
+impl AutoSyncSuppressionGuard {
+    pub fn new() -> Self {
+        AUTO_SYNC_SUPPRESS_DEPTH.fetch_add(1, Ordering::SeqCst);
+        Self
+    }
+}
+
+impl Drop for AutoSyncSuppressionGuard {
+    fn drop(&mut self) {
+        let _ =
+            AUTO_SYNC_SUPPRESS_DEPTH.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |value| {
+                Some(value.saturating_sub(1))
+            });
+    }
+}
+
+pub(crate) fn is_auto_sync_suppressed() -> bool {
+    AUTO_SYNC_SUPPRESS_DEPTH.load(Ordering::SeqCst) > 0
+}
+
+pub fn should_trigger_for_table(table: &str) -> bool {
+    let normalized = table.trim().to_ascii_lowercase();
+    matches!(
+        normalized.as_str(),
+        "providers"
+            | "provider_endpoints"
+            | "mcp_servers"
+            | "prompts"
+            | "skills"
+            | "skill_repos"
+            | "settings"
+            | "proxy_config"
+    )
+}
+
+pub(crate) fn enqueue_change_signal(tx: &Sender<String>, table: &str) -> bool {
+    match tx.try_send(table.to_string()) {
+        Ok(()) => true,
+        Err(TrySendError::Full(_)) | Err(TrySendError::Closed(_)) => false,
+    }
+}
+
+pub(crate) fn auto_sync_wait_duration(started_at: Instant, now: Instant) -> Option<Duration> {
+    let max_wait = Duration::from_millis(MAX_AUTO_SYNC_WAIT_MS);
+    let debounce = Duration::from_millis(AUTO_SYNC_DEBOUNCE_MS);
+    let elapsed = now.saturating_duration_since(started_at);
+    if elapsed >= max_wait {
+        return None;
+    }
+    Some(debounce.min(max_wait - elapsed))
+}
+
+fn should_run_auto_sync(settings: Option<&S3SyncSettings>) -> bool {
+    let Some(sync) = settings else {
+        return false;
+    };
+    sync.enabled && sync.auto_sync
+}
+
+fn persist_auto_sync_error(settings: &mut S3SyncSettings, error: &AppError) {
+    settings.status.last_error = Some(error.to_string());
+    settings.status.last_error_source = Some("auto".to_string());
+    let _ = settings::update_s3_sync_status(settings.status.clone());
+}
+
+fn emit_auto_sync_status_updated(app: &AppHandle, status: &str, error: Option<&str>) {
+    let payload = match error {
+        Some(message) => json!({
+            "source": "auto",
+            "status": status,
+            "error": message,
+        }),
+        None => json!({
+            "source": "auto",
+            "status": status,
+        }),
+    };
+
+    if let Err(err) = app.emit("s3-sync-status-updated", payload) {
+        log::debug!("[S3] failed to emit sync status update event: {err}");
+    }
+}
+
+async fn run_auto_sync_upload(
+    db: &crate::database::Database,
+    app: &AppHandle,
+) -> Result<(), AppError> {
+    let mut settings = settings::get_s3_sync_settings();
+    if !should_run_auto_sync(settings.as_ref()) {
+        return Ok(());
+    }
+
+    let mut sync_settings = match settings.take() {
+        Some(value) => value,
+        None => return Ok(()),
+    };
+
+    let result = s3_sync::run_with_sync_lock(s3_sync::upload(db, &mut sync_settings)).await;
+    match result {
+        Ok(_) => {
+            emit_auto_sync_status_updated(app, "success", None);
+            Ok(())
+        }
+        Err(err) => {
+            persist_auto_sync_error(&mut sync_settings, &err);
+            emit_auto_sync_status_updated(app, "error", Some(&err.to_string()));
+            Err(err)
+        }
+    }
+}
+
+pub fn notify_db_changed(table: &str) {
+    if is_auto_sync_suppressed() {
+        return;
+    }
+    if !should_trigger_for_table(table) {
+        return;
+    }
+    let Some(tx) = DB_CHANGE_TX.get() else {
+        return;
+    };
+    let _ = enqueue_change_signal(tx, table);
+}
+
+pub fn start_worker(db: Arc<crate::database::Database>, app: tauri::AppHandle) {
+    if DB_CHANGE_TX.get().is_some() {
+        return;
+    }
+
+    // Buffer size 1 is enough: we only need "dirty" signals, not every event.
+    let (tx, rx) = channel::<String>(1);
+    if DB_CHANGE_TX.set(tx).is_err() {
+        return;
+    }
+
+    tauri::async_runtime::spawn(async move {
+        run_worker_loop(db, rx, app).await;
+    });
+}
+
+async fn run_worker_loop(
+    db: Arc<crate::database::Database>,
+    mut rx: Receiver<String>,
+    app: tauri::AppHandle,
+) {
+    while let Some(first_table) = rx.recv().await {
+        let started_at = Instant::now();
+        let mut merged_count = 1usize;
+
+        while let Some(wait_for) = auto_sync_wait_duration(started_at, Instant::now()) {
+            let timeout = tokio::time::timeout(wait_for, rx.recv()).await;
+
+            match timeout {
+                Ok(Some(_)) => merged_count += 1,
+                Ok(None) => return,
+                Err(_) => break,
+            }
+        }
+
+        log::debug!(
+            "[S3][AutoSync] Triggered by table={first_table}, merged_changes={merged_count}"
+        );
+
+        if let Err(err) = run_auto_sync_upload(&db, &app).await {
+            log::warn!("[S3][AutoSync] Upload failed: {err}");
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        auto_sync_wait_duration, enqueue_change_signal, is_auto_sync_suppressed,
+        should_run_auto_sync, should_trigger_for_table, AutoSyncSuppressionGuard,
+        MAX_AUTO_SYNC_WAIT_MS,
+    };
+    use crate::settings::S3SyncSettings;
+    use std::time::{Duration, Instant};
+    use tokio::sync::mpsc::channel;
+
+    #[test]
+    fn should_trigger_sync_for_config_tables_only() {
+        assert!(should_trigger_for_table("providers"));
+        assert!(should_trigger_for_table("settings"));
+        assert!(!should_trigger_for_table("proxy_request_logs"));
+        assert!(!should_trigger_for_table("provider_health"));
+    }
+
+    #[test]
+    fn suppression_guard_enables_and_restores_state() {
+        assert!(!is_auto_sync_suppressed());
+        {
+            let _guard = AutoSyncSuppressionGuard::new();
+            assert!(is_auto_sync_suppressed());
+        }
+        assert!(!is_auto_sync_suppressed());
+    }
+
+    #[test]
+    fn max_wait_caps_flush_latency_for_continuous_events() {
+        let started = Instant::now();
+        let later = started + Duration::from_millis(MAX_AUTO_SYNC_WAIT_MS + 1);
+        assert!(auto_sync_wait_duration(started, later).is_none());
+    }
+
+    #[tokio::test]
+    async fn enqueue_change_signal_drops_when_channel_is_full() {
+        let (tx, _rx) = channel::<String>(1);
+        assert!(enqueue_change_signal(&tx, "providers"));
+        assert!(!enqueue_change_signal(&tx, "providers"));
+    }
+
+    #[test]
+    fn should_run_auto_sync_requires_enabled_and_auto_sync_flag() {
+        assert!(!should_run_auto_sync(None));
+
+        let disabled = S3SyncSettings {
+            enabled: false,
+            auto_sync: true,
+            ..S3SyncSettings::default()
+        };
+        assert!(!should_run_auto_sync(Some(&disabled)));
+
+        let auto_sync_off = S3SyncSettings {
+            enabled: true,
+            auto_sync: false,
+            ..S3SyncSettings::default()
+        };
+        assert!(!should_run_auto_sync(Some(&auto_sync_off)));
+
+        let enabled = S3SyncSettings {
+            enabled: true,
+            auto_sync: true,
+            ..S3SyncSettings::default()
+        };
+        assert!(should_run_auto_sync(Some(&enabled)));
+    }
+
+    #[test]
+    fn service_layer_does_not_depend_on_commands_layer() {
+        let source = include_str!("s3_auto_sync.rs");
+        let needle = ["crate", "commands", ""].join("::");
+        assert!(
+            !source.contains(&needle),
+            "services layer should not depend on commands layer"
+        );
+    }
+}