add concurrent thread config

assets-updater/src/core/config.rs

@@ -63,10 +63,6 @@ pub struct ExecutionConfig {
     pub proxy: Option<String>,
     pub timeout_seconds: u64,
     pub allow_cancel: bool,
-    /// How many successful downloads to accumulate before flushing the download
-    /// record to disk mid-run.  Set to `0` to disable mid-run flushing (record
-    /// is only written once at the end).  Mirrors Go's `batchSaveSize`.
-    pub batch_save_size: usize,
     pub retry: RetryConfig,
 }
 
@@ -76,7 +72,6 @@ impl Default for ExecutionConfig {
             proxy: None,
             timeout_seconds: 300,
             allow_cancel: true,
-            batch_save_size: 50,
             retry: RetryConfig::default(),
         }
     }

client/src/config.rs

@@ -16,5 +16,6 @@ pub struct Profile {
     #[serde(flatten)]
     pub sync_context: SyncContext,
     pub path: String,
-    pub interval: Option<u64>
+    pub interval: Option<u64>,
+    pub concurrent: Option<usize>,
 }

client/src/task.rs

@@ -29,6 +29,7 @@ pub async fn run(
         .await?,
     );
     let manifest_snapshot = { local_manifest.manifest.read().await.clone() };
+    let all_cnt = sync_resp.tasks.len();
     let tasks = sync_resp
         .tasks
         .into_iter()
@@ -40,8 +41,20 @@ pub async fn run(
             }
         })
         .collect::<Vec<_>>();
-    info!("[{}]: Collected {} tasks", profile.0, tasks.len());
-    let n = 5;
+    info!(
+        "[{}]: Collected {}/{} tasks",
+        profile.0,
+        tasks.len(),
+        all_cnt
+    );
+    if tasks.is_empty() {
+        info!("[{}]: No tasks to sync, skipping", profile.0);
+        let req = CloseRequest { id: id.clone() };
+        close(&mut client.get_client().await?, &req).await?;
+        return Ok(true);
+    }
+    let n = profile.1.concurrent.unwrap_or(5);
+    info!("[{}]: Start sync with {} thread", profile.0, n);
     let semaphore = Arc::new(Semaphore::new(n));
     let mut join_set = JoinSet::new();
     for task in tasks {

sekai-unpacker-client.yaml

@@ -16,7 +16,8 @@ profiles:
     filters:
       start_app:
         - "thumbnail"
-      on_demand: [ ]
+      on_demand:
+        - "thumbnail"
       skip: [ ]
       file_ext: [ ]
     export:
@@ -43,10 +44,12 @@ profiles:
     path: "./data/cn"
   jp:
     region: jp
+    concurrent: 50
     filters:
       start_app:
         - "thumbnail"
-      on_demand: [ ]
+      on_demand:
+        - "thumbnail"
       skip: [ ]
       file_ext: [ ]
     asset_version: 6.4.0.30

sekai-unpacker-server.yaml

@@ -12,7 +12,6 @@ execution:
   proxy: ""
   timeout_seconds: 300
   allow_cancel: true
-  batch_save_size: 50
   retry:
     attempts: 4
     initial_backoff_ms: 1000