fix tons of bugs

Cargo.lock

@@ -122,6 +122,7 @@ dependencies = [
  "tempfile",
  "thiserror 2.0.18",
  "tokio",
+ "twox-hash",
  "yaml_serde",
 ]
 
@@ -388,9 +389,11 @@ dependencies = [
  "bytes",
  "common",
  "communicator",
+ "futures-util",
  "h2",
  "lazy_static",
  "log",
+ "reqwest",
  "serde",
  "serde_json",
  "simplelog",
@@ -2795,6 +2798,15 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
+[[package]]
+name = "twox-hash"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+dependencies = [
+ "rand",
+]
+
 [[package]]
 name = "typenum"
 version = "1.19.0"

Cargo.toml

@@ -34,3 +34,4 @@ lazy_static = "1.5.0"
 structopt = "0.3.26"
 tokio-util = "0.7.18"
 futures-util = "0.3.32"
+twox-hash = "2.1.2"

assets-updater/Cargo.toml

@@ -22,3 +22,4 @@ cridecoder = { workspace = true }
 thiserror = { workspace = true }
 log = { workspace = true }
 cipher = { workspace = true, features = ["block-padding"] }
+twox-hash = { workspace = true, features = ["xxhash3_128"] }

assets-updater/src/core/asset_execution.rs

@@ -262,13 +262,17 @@ impl AssetExecutionContext {
             }
             RegionProviderConfig::Nuverse {
                 asset_version_url,
-                app_version,
                 asset_info_url_template,
                 ..
             } => {
                 // For nuverse, always fetch the version from asset_version_url.
                 // The incoming request.asset_version is intentionally ignored here
                 // to match Go reference behavior.
+                let app_version = self.sync_context.app_version.as_ref().ok_or_else(|| {
+                    AssetExecutionError::MissingAppVersion {
+                        region: self.region_name.clone(),
+                    }
+                })?;
                 let version_url = asset_version_url.replace("{app_version}", app_version);
                 let resolved_version =
                     String::from_utf8_lossy(&self.get_with_retry(&version_url).await?)
@@ -319,9 +323,13 @@ impl AssetExecutionContext {
             }
             RegionProviderConfig::Nuverse {
                 asset_bundle_url_template,
-                app_version,
                 ..
             } => {
+                let app_version = self.sync_context.app_version.as_ref().ok_or_else(|| {
+                    AssetExecutionError::MissingAppVersion {
+                        region: self.region_name.clone(),
+                    }
+                })?;
                 let asset_version = self
                     .resolved_asset_version
                     .as_deref()
@@ -435,6 +443,7 @@ impl AssetExecutionContext {
             &self.region,
             &temp_file,
             &task.bundle_path,
+            &task.download_path,
             category,
         )
         .await;

assets-updater/src/core/config.rs

@@ -150,7 +150,6 @@ pub enum RegionProviderConfig {
     },
     Nuverse {
         asset_version_url: String,
-        app_version: String,
         asset_info_url_template: String,
         asset_bundle_url_template: String,
         #[serde(default)]

assets-updater/src/core/errors.rs

@@ -107,6 +107,8 @@ pub enum AssetExecutionError {
     HttpStatus { url: String, status: u16 },
     #[error("region `{region}` is missing asset_save_dir")]
     MissingAssetSaveDir { region: String },
+    #[error("nuverse region `{region}` requires asset_version and asset_hash")]
+    MissingAppVersion { region: String },
     #[error("colorful_palette region `{region}` requires asset_version and asset_hash")]
     MissingAssetVersionOrHash { region: String },
     #[error("colorful_palette region `{region}` is missing profile hash for `{profile}`")]

assets-updater/src/core/export_pipeline.rs

@@ -54,18 +54,26 @@ pub fn get_export_group(export_path: &str) -> &'static str {
     "container"
 }
 
+pub fn get_hex_index(input: &str) -> String {
+    let hash_val = twox_hash::XxHash3_128::oneshot(input.as_bytes());
+    format!("{:032x}", hash_val)
+}
+
 pub async fn extract_unity_asset_bundle(
     app_config: &AppConfig,
     sync_context: &SyncContext,
     region: &RegionConfig,
     asset_bundle_file: &Path,
     export_path: &str,
+    download_path: &str,
     category: &str,
 ) -> Result<(PathBuf, bool), ExportPipelineError> {
+    let hash = get_hex_index(export_path);
     let output_dir = std::env::temp_dir()
         .join("sekai-updater")
         .join("extract")
-        .join(&sync_context.region);
+        .join(&sync_context.region)
+        .join(hash);
     let Some(asset_studio_cli_path) = app_config.tools.asset_studio_cli_path.as_deref() else {
         return Ok((asset_bundle_file.parent().unwrap().to_path_buf(), true));
     };
@@ -82,7 +90,7 @@ pub async fn extract_unity_asset_bundle(
     };
 
     let actual_export_path = if sync_context.export.by_category {
-        output_dir.join(category.to_lowercase()).join(export_path)
+        output_dir.join(category.to_lowercase())
     } else {
         output_dir.join(export_path)
     };
@@ -126,10 +134,23 @@ pub async fn extract_unity_asset_bundle(
     .await?;
 
     let mut count = 0;
-    walk(&actual_export_path, &mut |_| count += 1)?;
+    let mut is_bundle_name_file = true;
+    let download_name = match download_path.rsplit_once("/") {
+        None => download_path,
+        Some((_, name)) => name,
+    };
+    walk(&output_dir, &mut |f| {
+        count += 1;
+        let file_name = f.file_name().unwrap().to_string_lossy();
+        is_bundle_name_file = is_bundle_name_file
+            && match file_name.rsplit_once(".") {
+                Some((name, _)) => name.eq_ignore_ascii_case(download_name),
+                None => file_name.eq_ignore_ascii_case(download_name),
+            }
+    })?;
 
-    post_process_exported_files(app_config, sync_context, region, &actual_export_path).await?;
+    post_process_exported_files(app_config, sync_context, region, &output_dir).await?;
-    Ok((actual_export_path, count <= 1))
+    Ok((output_dir, is_bundle_name_file && count <= 1))
 }
 
 pub async fn post_process_exported_files(

client/Cargo.toml

@@ -19,3 +19,5 @@ anyhow = { workspace = true }
 h2 = { workspace = true }
 serde_json = { workspace = true }
 tokio-util = { workspace = true }
+reqwest = { workspace = true }
+futures-util = { workspace = true }

client/src/config.rs

@@ -11,6 +11,12 @@ pub struct ClientConfig {
     pub profiles: HashMap<String, Profile>,
 }
 
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct DynamicLoad {
+    pub url: String,
+    pub map: HashMap<String, String>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Profile {
     #[serde(flatten)]
@@ -18,4 +24,51 @@ pub struct Profile {
     pub path: String,
     pub interval: Option<u64>,
     pub concurrent: Option<usize>,
+    dynamic_load: Option<DynamicLoad>,
+}
+
+impl Profile {
+    pub fn need_update(&self) -> bool {
+        self.dynamic_load.is_some()
+    }
+
+    pub async fn update(&mut self) -> anyhow::Result<()> {
+        let Some(load) = &self.dynamic_load else {
+            return Ok(());
+        };
+        let response = reqwest::get(&load.url).await?;
+        let body = response.text().await?;
+        let data: serde_json::Value = serde_json::from_str(&body)?;
+        if !data.is_object() {
+            return Err(anyhow::Error::msg("response is not an object"));
+        };
+        let obj = data.as_object().unwrap();
+        macro_rules! load_field {
+            (from $obj:expr; $($tail:tt)*) => {
+                load_field!(@munch $obj, $($tail)*);
+            };
+            (@munch $obj:expr, $field:ident$(.$f_more:ident)* = $key:ident as $t:ident $($rest:tt)*) => {
+                load_field!(@suffix $obj, ($field$(.$f_more)*), $key, $t, [], $($rest)*);
+            };
+            (@suffix $obj:expr, ($($f_full:tt)*), $key:ident, $t:ident, [$($acc:tt)*], ; $($rest:tt)*) => {
+                if let Some(k) = load.map.get(stringify!($key)) {
+                    if let Some(serde_json::Value::$t(v)) = $obj.get(k) {
+                        $($f_full)* = Some(v.clone() $($acc)*);
+                    }
+                }
+                load_field!(@munch $obj, $($rest)*);
+            };
+            (@suffix $obj:expr, ($($f_full:tt)*), $key:ident, $t:ident, [$($acc:tt)*], $token:tt $($rest:tt)*) => {
+                load_field!(@suffix $obj, ($($f_full)*), $key, $t, [$($acc)* $token], $($rest)*);
+            };
+            (@munch $obj:expr, ) => {};
+        }
+        load_field! {
+            from obj;
+            self.sync_context.asset_version = asset_version as String;
+            self.sync_context.asset_hash = asset_hash as String;
+            self.concurrent = concurrent as Number.as_u64().ok_or(anyhow::Error::msg("concurrent is not usize"))? as usize;
+        }
+        Ok(())
+    }
 }

client/src/main.rs

@@ -2,16 +2,17 @@ use crate::config::{ClientConfig, Profile};
 use crate::task::run;
 use common::strings::REGION_NOT_FOUND;
 use communicator::{ClientManager, Identity, TunnelEndpoint, TunnelListener, connect_tunnel};
+use futures_util::future::join_all;
 use lazy_static::lazy_static;
 use log::{LevelFilter, error, info};
 use simplelog::{ColorChoice, Config, TermLogger, TerminalMode};
 use std::collections::{HashMap, VecDeque};
 use std::fs;
 use std::str::FromStr;
-use std::sync::{Arc, RwLock};
+use std::sync::Arc;
 use std::time::Duration;
 use structopt::StructOpt;
-use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc};
+use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore, mpsc};
 use tokio::task::JoinSet;
 use tokio::time::sleep;
 use tokio_util::sync::CancellationToken;
@@ -51,16 +52,22 @@ async fn main() -> anyhow::Result<()> {
         std::process::exit(1);
     }
 
-    let profiles = command_opts
+    let profiles = join_all(command_opts.profile.iter().map(async |p| {
-        .profile
-        .iter()
-        .map(|p| {
         let profile = CONFIG.profiles.get(p.as_str());
         match profile {
-                Some(profile) => Ok((p.to_string(), profile.clone())),
+            Some(profile) => {
+                let mut profile = profile.clone();
+                if profile.need_update() {
+                    info!("Loading async config of {}", p);
+                    profile.update().await?;
+                }
+                Ok((p.to_string(), Arc::new(RwLock::new(profile))))
+            }
             None => Err(anyhow::anyhow!("Profile `{}` not found in config", p)),
         }
-        })
+    }))
+    .await
+    .into_iter()
     .collect::<Result<Vec<_>, _>>()?;
 
     let mut join_set = JoinSet::new();
@@ -121,15 +128,25 @@ async fn main() -> anyhow::Result<()> {
         let semaphore = Arc::new(Semaphore::new(1));
         let cancel_token = CancellationToken::new();
         let post_task = {
-            async |profile: Arc<(String, Profile)>,
+            async |profile: Arc<(String, Arc<RwLock<Profile>>)>,
                    permit: OwnedSemaphorePermit,
                    cancel_token: CancellationToken| {
-                match profile.1.interval {
+                let (interval, need_update) = {
+                    let p = profile.1.read().await;
+                    (p.interval, p.need_update())
+                };
+                match interval {
                     None => {
                         cancel_token.cancel();
                     }
                     Some(interval) => {
                         sleep(Duration::from_secs(interval)).await;
+                        if need_update {
+                            info!("Trying to update profile for {}", profile.0);
+                            if let Err(e) = profile.1.write().await.update().await {
+                                error!("Failed to update profile for {}: {}", profile.0, e);
+                            }
+                        }
                     }
                 }
                 drop(permit);
@@ -263,13 +280,13 @@ async fn main() -> anyhow::Result<()> {
 }
 
 struct Sender<T> {
-    inner: RwLock<VecDeque<T>>,
+    inner: std::sync::RwLock<VecDeque<T>>,
 }
 
 impl<T> Sender<T> {
     pub fn new() -> Self {
         Self {
-            inner: RwLock::new(VecDeque::new()),
+            inner: std::sync::RwLock::new(VecDeque::new()),
         }
     }
 

client/src/task.rs

@@ -6,27 +6,22 @@ use log::{error, info};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc};
 use tokio::sync::{RwLock, Semaphore};
 use tokio::task::JoinSet;
 
 pub async fn run(
     client: Arc<ClientManager>,
-    profile: Arc<(String, Profile)>,
+    profile: Arc<(String, Arc<RwLock<Profile>>)>,
 ) -> anyhow::Result<bool> {
     info!("[{}]: Starting sync", profile.0);
-    tokio::fs::create_dir_all(&profile.1.path).await?;
+    let p1 = Arc::new(profile.1.read().await.clone());
-    let sync_resp = sync(&mut client.get_client().await?, &profile.1).await?;
+    tokio::fs::create_dir_all(&p1.path).await?;
+    let sync_resp = sync(&mut client.get_client().await?, &p1).await?;
     let id = sync_resp.id;
     let local_manifest = Arc::new(
-        AutoSaveManifest::new(
+        AutoSaveManifest::new(5, Path::new(&p1.path).join("manifest.json").to_path_buf()).await?,
-            5,
-            Path::new(&profile.1.path)
-                .join("manifest.json")
-                .to_path_buf(),
-        )
-        .await?,
     );
     let manifest_snapshot = { local_manifest.manifest.read().await.clone() };
     let all_cnt = sync_resp.tasks.len();
@@ -53,7 +48,7 @@ pub async fn run(
         close(&mut client.get_client().await?, &req).await?;
         return Ok(true);
     }
-    let n = profile.1.concurrent.unwrap_or(5);
+    let n = p1.concurrent.unwrap_or(5);
     info!("[{}]: Start sync with {} thread", profile.0, n);
     let semaphore = Arc::new(Semaphore::new(n));
     let mut join_set = JoinSet::new();
@@ -62,18 +57,18 @@ pub async fn run(
         let client = client.clone();
         let id = id.clone();
         let local_manifest = local_manifest.clone();
-        let profile = profile.clone();
+        let p1 = p1.clone();
 
         join_set.spawn(async move {
             let req = DownloadRequest {
                 id: id.clone(),
                 task: task.clone(),
             };
-            let result = download(&mut client.get_client().await.unwrap(), &req, &profile.1).await;
+            let result = download(&mut client.get_client().await.unwrap(), &req, &p1).await;
             if let Err(e) = result
                 && let Some(_) = e.downcast_ref::<h2::Error>()
             {
-                download(&mut client.get_client().await.unwrap(), &req, &profile.1)
+                download(&mut client.get_client().await.unwrap(), &req, &p1)
                     .await
                     .unwrap();
             }

common/src/updater.rs

@@ -12,6 +12,8 @@ pub struct SyncContext {
     #[serde(default)]
     pub asset_hash: Option<String>,
     #[serde(default)]
+    pub app_version: Option<String>,
+    #[serde(default)]
     pub exact_single_file_bundle: bool,
 }
 

sekai-unpacker-client.yaml

@@ -1,19 +1,20 @@
 log_level: "DEBUG"
 
-#client:
+client:
-#  - url: "127.0.0.1:3333"
+  - url: "127.0.0.1:3333"
-#    token: abc
-server:
-  - url: 127.0.0.1:3333
     token: abc
-    cert: "D:\\WorkDir\\Nginx\\cert\\_.bluemangoo.net\\_.bluemangoo.net-chain.pem"
+#server:
-    key: "D:\\WorkDir\\Nginx\\cert\\_.bluemangoo.net\\_.bluemangoo.net-key.pem"
+#  - url: 127.0.0.1:3333
+#    token: abc
+#    cert: "D:\\WorkDir\\Nginx\\cert\\_.bluemangoo.net\\_.bluemangoo.net-chain.pem"
+#    key: "D:\\WorkDir\\Nginx\\cert\\_.bluemangoo.net\\_.bluemangoo.net-key.pem"
 
 profiles:
   cn:
     region: cn
     # interval: 3 # seconds
     concurrent: 50
+    app_version: "6.0.0"
     filters:
       start_app:
         - "thumbnail"
@@ -44,7 +45,7 @@ profiles:
         convert_to_mp3: false
         convert_to_flac: false
         remove_wav: false
-#    exact_single_file_bundle: true
+    exact_single_file_bundle: true
     path: "./data/cn"
   jp:
     region: jp
@@ -58,8 +59,11 @@ profiles:
         - "stamp"
       skip: [ ]
       file_ext: [ ]
-    asset_version: 6.4.0.30
+    dynamic_load:
-    asset_hash: cce60d07-d60e-48dd-be22-12eac2e67950
+      url: "https://github.com/Team-Haruki/haruki-sekai-master/raw/refs/heads/main/versions/current_version.json"
+      map:
+        asset_version: assetVersion
+        asset_hash: assetHash
     export:
       by_category: false
       usm:
@@ -81,5 +85,5 @@ profiles:
         convert_to_mp3: false
         convert_to_flac: false
         remove_wav: false
-#    exact_single_file_bundle: true
+    exact_single_file_bundle: true
     path: "./data/jp"

sekai-unpacker-server.yaml

@@ -1,12 +1,12 @@
 log_level: "DEBUG"
 
-client:
+#client:
-  - url: "127.0.0.1:3333"
+#  - url: "127.0.0.1:3333"
-    token: abc
-    host: "local.bluemangoo.net"
-#server:
-#  - url: 127.0.0.1:3333
 #    token: abc
+#    host: "local.bluemangoo.net"
+server:
+  - url: 127.0.0.1:3333
+    token: abc
 
 execution:
   proxy: ""
@@ -49,7 +49,6 @@ regions:
     provider:
       kind: nuverse
       asset_version_url: "https://lf3-mkcncdn-tos.dailygn.com/obj/rt-game-lf/gdl_app_5236/Mainland/{app_version}/Release/cn_online/ios/version"
-      app_version: "5.2.0"
       asset_info_url_template: "https://lf3-mkcncdn-tos.dailygn.com/obj/sf-game-lf/gdl_app_5236/AssetBundle/{app_version}/Release/cn_online/ios{asset_version}/AssetBundleInfoNew.json"
       asset_bundle_url_template: "https://lf3-mkcncdn-tos.dailygn.com/obj/sf-game-lf/gdl_app_5236/AssetBundle/{app_version}/Release/cn_online/{bundle_path}"
       required_cookies: false

server/src/router/download.rs

@@ -7,6 +7,7 @@ use communicator::http::{json_from_request, send, send_error};
 use h2::RecvStream;
 use h2::server::SendResponse;
 use http::{Request, Response};
+use log::debug;
 
 pub async fn download(
     mut request: Request<RecvStream>,
@@ -43,32 +44,6 @@ pub async fn download(
     }
     let (dir, single_file) = dir.unwrap();
 
-    let dir_base = std::env::temp_dir()
-        .join("sekai-updater")
-        .join("extract")
-        .join(&context.sync_context.region);
-
-    let files = if !dir.is_dir() {
-        if context.sync_context.filters.file_ext.is_empty()
-            || dir.extension().is_some_and(|t| {
-                context
-                    .sync_context
-                    .filters
-                    .file_ext
-                    .contains(&t.to_str().unwrap().to_lowercase())
-            })
-        {
-            vec![(
-                dir.clone(),
-                dir.strip_prefix(&dir_base)
-                    .unwrap()
-                    .to_string_lossy()
-                    .to_string(),
-            )]
-        } else {
-            vec![]
-        }
-    } else {
     let files = if context.sync_context.filters.file_ext.is_empty() {
         find_files(&dir)
     } else {
@@ -78,23 +53,28 @@ pub async fn download(
         send_error(send_response, error.into());
         return Ok(());
     }
-        let base = dir.strip_prefix(&dir_base).unwrap();
+    let files = files
-        let base_str = if single_file {
-            base.parent().unwrap().to_str().unwrap()
-        } else {
-            base.to_str().unwrap()
-        };
-        files
         .unwrap()
         .iter()
         .map(|p| {
             let raw = p.clone();
             let name = p.strip_prefix(&dir).unwrap();
-                let path = format!("{}/{}", base_str, name.to_string_lossy());
+            debug!("{} {}", p.to_string_lossy(), single_file);
+            let path = if context.sync_context.exact_single_file_bundle
+                && single_file
+            {
+                format!(
+                    "{}/{}",
+                    name.parent().unwrap().parent().unwrap().to_string_lossy(),
+                    p.file_name().unwrap().to_string_lossy()
+                )
+            } else {
+                name.to_string_lossy().to_string()
+            };
+
             (raw, path)
         })
-            .collect::<Vec<_>>()
+        .collect::<Vec<_>>();
-    };
 
     let response = Response::builder()
         .status(200)