Bluemangoo/sekai-unpacker
1
0
Fork 0
mirror of https://github.com/Bluemangoo/sekai-unpacker.git synced 2026-09-20 00:06:52 +08:00
Code Activity

Initial Commit

Browse Source
This commit is contained in:
Bluemangoo
2026-04-10 15:23:22 +08:00
commit 7accb43048
40 changed files with 7950 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
assets-updater/Cargo.toml
+24
View File
@@ -0,0 +1,24 @@
[package]
name = "assets-updater"
version = "0.1.0"
edition = "2024"
[dependencies]
common = { path = "../common" }
aes = { workspace = true }
tokio = { workspace = true, features = ["process"] }
rmp-serde = { workspace = true }
hex = { workspace = true }
cbc = { workspace = true }
tempfile = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
reqwest = { workspace = true }
serde = { workspace = true, features = ["derive"] }
regex = { workspace = true }
yaml_serde = { workspace = true }
image = { workspace = true }
cridecoder = { workspace = true }
thiserror = { workspace = true }
log = { workspace = true }
cipher = { workspace = true, features = ["block-padding"] }
assets-updater/src/core/asset_execution.rs
+576
View File
@@ -0,0 +1,576 @@
use std::collections::HashMap;
use std::path::PathBuf;
use std::time::Duration;
use crate::core::config::{AppConfig, RegionConfig, RegionProviderConfig};
use crate::core::errors::AssetExecutionError;
use crate::core::export_pipeline::extract_unity_asset_bundle;
use crate::core::regions::{compile_patterns, matches_any};
use crate::core::retry::retry_async;
use aes::cipher::block_padding::Pkcs7;
use aes::cipher::{BlockDecryptMut, KeyIvInit};
use chrono::FixedOffset;
use common::updater::{AssetCategory, DownloadTask, SyncContext};
use reqwest::header::{
ACCEPT, ACCEPT_ENCODING, ACCEPT_LANGUAGE, CONNECTION, COOKIE, HeaderMap, HeaderValue,
SET_COOKIE, USER_AGENT,
};
use serde::{Deserialize, Serialize};
type Aes128CbcDec = cbc::Decryptor<aes::Aes128>;
type Aes192CbcDec = cbc::Decryptor<aes::Aes192>;
type Aes256CbcDec = cbc::Decryptor<aes::Aes256>;
/// Deserializes a msgpack/JSON null or missing value as an empty String.
/// Go silently coerces nil → zero value for non-pointer types; this helper
/// mirrors that behavior for String fields.
fn de_null_as_empty_string<'de, D>(deserializer: D) -> Result<String, D::Error>
where
D: serde::Deserializer<'de>,
{
Ok(Option::<String>::deserialize(deserializer)?.unwrap_or_default())
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AssetBundleDetail {
#[serde(rename = "bundleName", deserialize_with = "de_null_as_empty_string")]
pub bundle_name: String,
#[serde(rename = "cacheFileName", deserialize_with = "de_null_as_empty_string")]
pub cache_file_name: String,
#[serde(
rename = "cacheDirectoryName",
deserialize_with = "de_null_as_empty_string"
)]
pub cache_directory_name: String,
// nuverse regions use `crc` instead of `hash`; the server may send nil here.
#[serde(rename = "hash", deserialize_with = "de_null_as_empty_string")]
pub hash: String,
#[serde(rename = "category")]
pub category: AssetCategory,
#[serde(rename = "crc")]
pub crc: i64,
#[serde(rename = "fileSize")]
pub file_size: i64,
#[serde(rename = "dependencies")]
pub dependencies: Vec<String>,
#[serde(rename = "paths", default)]
pub paths: Vec<String>,
#[serde(rename = "isBuiltin")]
pub is_builtin: bool,
#[serde(rename = "isRelocate")]
pub is_relocate: Option<bool>,
#[serde(rename = "md5Hash")]
pub md5_hash: Option<String>,
#[serde(rename = "downloadPath")]
pub download_path: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AssetBundleInfo {
#[serde(rename = "version")]
pub version: Option<String>,
#[serde(rename = "os")]
pub os: Option<String>,
#[serde(rename = "bundles")]
pub bundles: HashMap<String, AssetBundleDetail>,
}
#[derive(Debug, Clone)]
pub struct AssetExecutionContext {
client: reqwest::Client,
region_name: String,
region: RegionConfig,
retry: crate::core::config::RetryConfig,
runtime_cookie: Option<String>,
resolved_asset_version: Option<String>,
pub sync_context: SyncContext,
}
impl AssetExecutionContext {
pub fn new(
app_config: &AppConfig,
sync_context: &SyncContext,
region: &RegionConfig,
) -> Result<Self, AssetExecutionError> {
let mut headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static("*/*"));
headers.insert(
USER_AGENT,
HeaderValue::from_static("ProductName/134 CFNetwork/1408.0.4 Darwin/22.5.0"),
);
headers.insert(CONNECTION, HeaderValue::from_static("keep-alive"));
headers.insert(
ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br"),
);
headers.insert(
ACCEPT_LANGUAGE,
HeaderValue::from_static("zh-CN,zh-Hans;q=0.9"),
);
headers.insert(
"X-Unity-Version",
HeaderValue::from_str(&region.runtime.unity_version)
.map_err(|err| AssetExecutionError::HttpClient(err.to_string()))?,
);
let mut builder = reqwest::Client::builder()
.default_headers(headers)
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(180))
.pool_max_idle_per_host(100)
.tcp_keepalive(Duration::from_secs(30));
if let Some(proxy) = &app_config.execution.proxy
&& !proxy.is_empty()
{
builder = builder.proxy(
reqwest::Proxy::all(proxy)
.map_err(|err| AssetExecutionError::HttpClient(err.to_string()))?,
);
}
Ok(Self {
client: builder
.build()
.map_err(|err| AssetExecutionError::HttpClient(err.to_string()))?,
sync_context: sync_context.clone(),
region_name: sync_context.region.clone(),
region: region.clone(),
retry: app_config.execution.retry.clone(),
runtime_cookie: None,
resolved_asset_version: sync_context.asset_version.clone(),
})
}
pub async fn fetch_tasks(&mut self) -> Result<Vec<DownloadTask>, AssetExecutionError> {
if self.requires_cookies() {
self.fetch_runtime_cookies().await?;
}
let info = self.fetch_asset_bundle_info().await?;
let tasks = self.build_download_tasks(&info);
Ok(tasks)
}
pub async fn download(
&self,
task: &DownloadTask,
app_config: &AppConfig,
) -> Result<PathBuf, AssetExecutionError> {
let ctx = self.clone();
ctx.download_and_export_bundle(app_config, task).await
}
fn requires_cookies(&self) -> bool {
match &self.region.provider {
RegionProviderConfig::ColorfulPalette {
required_cookies, ..
} => *required_cookies,
RegionProviderConfig::Nuverse {
required_cookies, ..
} => *required_cookies,
}
}
async fn fetch_runtime_cookies(&mut self) -> Result<(), AssetExecutionError> {
let url = match &self.region.provider {
RegionProviderConfig::ColorfulPalette {
cookie_bootstrap_url,
..
}
| RegionProviderConfig::Nuverse {
cookie_bootstrap_url,
..
} => cookie_bootstrap_url.clone().unwrap_or_else(|| {
"https://issue.sekai.colorfulpalette.org/api/signature".to_string()
}),
};
self.runtime_cookie = retry_async(
&self.retry,
"cookie bootstrap",
|_| async {
let response = self.client.post(&url).send().await?;
if response.status().is_success() {
Ok(response
.headers()
.get(SET_COOKIE)
.and_then(|value| value.to_str().ok())
.map(str::to_string))
} else {
Err(AssetExecutionError::HttpStatus {
url: url.clone(),
status: response.status().as_u16(),
})
}
},
is_retryable_http_error,
)
.await?;
Ok(())
}
async fn fetch_asset_bundle_info(&mut self) -> Result<AssetBundleInfo, AssetExecutionError> {
let url = self.render_asset_info_url().await?;
let body = self.get_with_retry(&url).await?;
decrypt_asset_bundle_info(
self.region.crypto.aes_key_hex.as_deref().ok_or_else(|| {
AssetExecutionError::MissingCryptoConfig {
region: self.region_name.clone(),
}
})?,
self.region.crypto.aes_iv_hex.as_deref().ok_or_else(|| {
AssetExecutionError::MissingCryptoConfig {
region: self.region_name.clone(),
}
})?,
&body,
)
}
async fn render_asset_info_url(&mut self) -> Result<String, AssetExecutionError> {
match &self.region.provider {
RegionProviderConfig::ColorfulPalette {
asset_info_url_template,
profile,
profile_hashes,
..
} => {
let asset_version =
self.sync_context.asset_version.as_deref().ok_or_else(|| {
AssetExecutionError::MissingAssetVersionOrHash {
region: self.region_name.clone(),
}
})?;
let asset_hash = self.sync_context.asset_hash.as_deref().ok_or_else(|| {
AssetExecutionError::MissingAssetVersionOrHash {
region: self.region_name.clone(),
}
})?;
let profile_hash = profile_hashes.get(profile).ok_or_else(|| {
AssetExecutionError::MissingProfileHash {
region: self.region_name.clone(),
profile: profile.clone(),
}
})?;
Ok(asset_info_url_template
.replace("{env}", profile)
.replace("{hash}", profile_hash)
.replace("{asset_version}", asset_version)
.replace("{asset_hash}", asset_hash)
+ &time_arg_jst())
}
RegionProviderConfig::Nuverse {
asset_version_url,
app_version,
asset_info_url_template,
..
} => {
// For nuverse, always fetch the version from asset_version_url.
// The incoming request.asset_version is intentionally ignored here
// to match Go reference behavior.
let version_url = asset_version_url.replace("{app_version}", app_version);
let resolved_version =
String::from_utf8_lossy(&self.get_with_retry(&version_url).await?)
.trim()
.to_string();
self.resolved_asset_version = Some(resolved_version.clone());
Ok(asset_info_url_template
.replace("{app_version}", app_version)
.replace("{asset_version}", &resolved_version)
+ &time_arg_jst())
}
}
}
fn render_bundle_url(&self, task: &DownloadTask) -> Result<String, AssetExecutionError> {
match &self.region.provider {
RegionProviderConfig::ColorfulPalette {
asset_bundle_url_template,
profile,
profile_hashes,
..
} => {
let asset_version =
self.sync_context.asset_version.as_deref().ok_or_else(|| {
AssetExecutionError::MissingAssetVersionOrHash {
region: self.region_name.clone(),
}
})?;
let asset_hash = self.sync_context.asset_hash.as_deref().ok_or_else(|| {
AssetExecutionError::MissingAssetVersionOrHash {
region: self.region_name.clone(),
}
})?;
let profile_hash = profile_hashes.get(profile).ok_or_else(|| {
AssetExecutionError::MissingProfileHash {
region: self.region_name.clone(),
profile: profile.clone(),
}
})?;
Ok(asset_bundle_url_template
.replace("{bundle_path}", &task.download_path)
.replace("{asset_version}", asset_version)
.replace("{asset_hash}", asset_hash)
.replace("{env}", profile)
.replace("{hash}", profile_hash)
+ &time_arg_jst())
}
RegionProviderConfig::Nuverse {
asset_bundle_url_template,
app_version,
..
} => {
let asset_version = self
.resolved_asset_version
.as_deref()
.unwrap_or("<resolved-asset-version>");
Ok(asset_bundle_url_template
.replace("{bundle_path}", &task.download_path)
.replace("{app_version}", app_version)
.replace("{asset_version}", asset_version)
+ &time_arg_jst())
}
}
}
async fn get_with_retry(&self, url: &str) -> Result<Vec<u8>, AssetExecutionError> {
retry_async(
&self.retry,
"http get",
|_| async {
let mut request = self.client.get(url);
if let Some(cookie) = &self.runtime_cookie {
request = request.header(COOKIE, cookie);
}
match request.send().await {
Ok(response) if response.status().is_success() => {
Ok(response.bytes().await?.to_vec())
}
Ok(response) => Err(AssetExecutionError::HttpStatus {
url: url.to_string(),
status: response.status().as_u16(),
}),
Err(err) => Err(AssetExecutionError::Http(err)),
}
},
is_retryable_http_error,
)
.await
}
fn build_download_tasks(&self, info: &AssetBundleInfo) -> Vec<DownloadTask> {
let skip_patterns = compile_patterns(&self.sync_context.filters.skip);
let start_app_patterns = compile_patterns(&self.sync_context.filters.start_app);
let on_demand_patterns = compile_patterns(&self.sync_context.filters.on_demand);
let mut tasks = Vec::new();
for (bundle_name, detail) in &info.bundles {
if matches_any(&skip_patterns, bundle_name) {
continue;
}
let category_patterns = match &detail.category {
AssetCategory::StartApp => &start_app_patterns,
AssetCategory::OnDemand => &on_demand_patterns,
AssetCategory::Other(_) => continue,
};
if category_patterns.is_empty() || !matches_any(category_patterns, bundle_name) {
continue;
}
let bundle_hash = match self.region.provider {
RegionProviderConfig::Nuverse { .. } => detail.crc.to_string(),
RegionProviderConfig::ColorfulPalette { .. } => detail.hash.clone(),
};
tasks.push(DownloadTask {
download_path: download_path_for_region(&self.region.provider, bundle_name, detail),
bundle_path: bundle_name.clone(),
bundle_hash,
category: detail.category.clone(),
});
}
tasks.sort_by(|a, b| a.bundle_path.cmp(&b.bundle_path));
tasks
}
async fn download_and_export_bundle(
&self,
app_config: &AppConfig,
task: &DownloadTask,
) -> Result<PathBuf, AssetExecutionError> {
let bundle_url = self.render_bundle_url(task)?;
let body = self.get_with_retry(&bundle_url).await?;
let deobfuscated = deobfuscate(&body);
let temp_file = std::env::temp_dir()
.join("sekai-updater")
.join("obf")
.join(&self.region_name)
.join(&task.bundle_path);
if let Some(parent) = temp_file.parent() {
std::fs::create_dir_all(parent).map_err(|source| {
AssetExecutionError::CreateTempDir {
path: parent.to_path_buf(),
source,
}
})?;
}
std::fs::write(&temp_file, deobfuscated).map_err(|source| {
AssetExecutionError::WriteTempFile {
path: temp_file.clone(),
source,
}
})?;
let category = match task.category {
AssetCategory::StartApp => "StartApp",
AssetCategory::OnDemand => "OnDemand",
AssetCategory::Other(_) => "OnDemand",
};
let export_result = extract_unity_asset_bundle(
app_config,
&self.sync_context,
&self.region,
&temp_file,
&task.bundle_path,
category,
)
.await;
let _ = std::fs::remove_file(&temp_file);
export_result.map_err(Into::into)
}
}
pub async fn fetch_live_asset_bundle_info(
app_config: &AppConfig,
sync_context: &SyncContext,
region: &RegionConfig,
) -> Result<AssetBundleInfo, AssetExecutionError> {
let mut context = AssetExecutionContext::new(app_config, sync_context, region)?;
if context.requires_cookies() {
context.fetch_runtime_cookies().await?;
}
context.fetch_asset_bundle_info().await
}
fn is_retryable_http_error(err: &AssetExecutionError) -> bool {
match err {
AssetExecutionError::Http(_) => true,
AssetExecutionError::HttpStatus { status, .. } => *status >= 500,
_ => false,
}
}
pub fn decrypt_asset_bundle_info(
aes_key_hex: &str,
aes_iv_hex: &str,
content: &[u8],
) -> Result<AssetBundleInfo, AssetExecutionError> {
if content.is_empty() {
return Err(AssetExecutionError::EmptyEncryptedContent);
}
if !content.len().is_multiple_of(16) {
return Err(AssetExecutionError::InvalidEncryptedBlockSize);
}
let key = hex::decode(aes_key_hex)
.map_err(|err| AssetExecutionError::InvalidAesKeyHex(err.to_string()))?;
let iv = hex::decode(aes_iv_hex)
.map_err(|err| AssetExecutionError::InvalidAesIvHex(err.to_string()))?;
if iv.len() != 16 {
return Err(AssetExecutionError::InvalidAesIvLength { got: iv.len() });
}
let mut buf = content.to_vec();
let decrypted = match key.len() {
16 => Aes128CbcDec::new_from_slices(&key, &iv)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?
.decrypt_padded_mut::<Pkcs7>(&mut buf)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?,
24 => Aes192CbcDec::new_from_slices(&key, &iv)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?
.decrypt_padded_mut::<Pkcs7>(&mut buf)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?,
32 => Aes256CbcDec::new_from_slices(&key, &iv)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?
.decrypt_padded_mut::<Pkcs7>(&mut buf)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?,
_ => {
return Err(AssetExecutionError::AssetInfoDecode(format!(
"unsupported AES key length {}",
key.len()
)));
}
};
rmp_serde::from_slice::<AssetBundleInfo>(decrypted)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))
}
pub fn deobfuscate(data: &[u8]) -> Vec<u8> {
const SIMPLE: [u8; 4] = [0x20, 0x00, 0x00, 0x00];
const XOR_HEADER: [u8; 4] = [0x10, 0x00, 0x00, 0x00];
if data.starts_with(&SIMPLE) {
return data[4..].to_vec();
}
if data.starts_with(&XOR_HEADER) {
let body = &data[4..];
if body.len() < 128 {
return body.to_vec();
}
let mut header = vec![0u8; 128];
let pattern = [0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00];
for idx in 0..128 {
header[idx] = body[idx] ^ pattern[idx % pattern.len()];
}
let mut output = header;
output.extend_from_slice(&body[128..]);
return output;
}
data.to_vec()
}
pub fn should_download_bundle(
sync_context: &SyncContext,
bundle_name: &str,
category: &AssetCategory,
) -> bool {
let compiled = match category {
AssetCategory::StartApp => compile_patterns(&sync_context.filters.start_app),
AssetCategory::OnDemand => compile_patterns(&sync_context.filters.on_demand),
AssetCategory::Other(_) => return false,
};
if compiled.is_empty() {
return false;
}
matches_any(&compiled, bundle_name)
}
fn download_path_for_region(
provider: &RegionProviderConfig,
bundle_name: &str,
detail: &AssetBundleDetail,
) -> String {
match provider {
RegionProviderConfig::ColorfulPalette { .. } => bundle_name.to_string(),
RegionProviderConfig::Nuverse { .. } => detail
.download_path
.as_ref()
.map(|prefix| format!("{prefix}/{bundle_name}"))
.unwrap_or_else(|| bundle_name.to_string()),
}
}
fn time_arg_jst() -> String {
let tz = FixedOffset::east_opt(9 * 3600).unwrap();
format!(
"?t={}",
chrono::Utc::now().with_timezone(&tz).format("%Y%m%d%H%M%S")
)
}
assets-updater/src/core/codec.rs
+89
View File
@@ -0,0 +1,89 @@
use std::fs::File;
use std::path::{Path, PathBuf};
use cridecoder::{extract_acb_from_file, extract_usm_file, HcaDecoder};
use serde::Serialize;
use crate::core::errors::CodecError;
pub const CODEC_BACKEND: &str = "crates.io:[email protected]";
#[derive(Debug, Clone, Serialize)]
pub struct CodecSummary {
pub backend: &'static str,
pub supports_acb: bool,
pub supports_usm: bool,
pub supports_hca_to_wav: bool,
pub supports_usm_metadata: bool,
}
pub fn codec_summary() -> CodecSummary {
CodecSummary {
backend: CODEC_BACKEND,
supports_acb: true,
supports_usm: true,
supports_hca_to_wav: true,
supports_usm_metadata: true,
}
}
pub fn export_acb(input: &Path, output_dir: &Path) -> Result<Option<Vec<String>>, CodecError> {
extract_acb_from_file(input, output_dir).map_err(|err| CodecError::Acb(err.to_string()))
}
pub fn export_usm(input: &Path, output_dir: &Path) -> Result<Vec<PathBuf>, CodecError> {
let outputs = extract_usm_file(input, output_dir, None, false)
.map_err(|err| CodecError::Usm(err.to_string()))?;
normalize_usm_output_names(input, outputs)
}
pub fn read_usm_metadata(input: &Path) -> Result<cridecoder::usm::Metadata, CodecError> {
cridecoder::usm::read_metadata_file(input).map_err(|err| CodecError::Metadata(err.to_string()))
}
pub fn decode_hca_to_wav(input: &Path, output: &Path) -> Result<(), CodecError> {
let input_path = input
.to_str()
.ok_or_else(|| CodecError::NonUtf8Path(input.to_path_buf()))?;
let mut decoder =
HcaDecoder::from_file(input_path).map_err(|err| CodecError::Hca(err.to_string()))?;
let mut file = File::create(output).map_err(|source| CodecError::Io {
path: output.to_path_buf(),
source,
})?;
decoder
.decode_to_wav(&mut file)
.map_err(|err| CodecError::Hca(err.to_string()))
}
fn normalize_usm_output_names(
input: &Path,
outputs: Vec<PathBuf>,
) -> Result<Vec<PathBuf>, CodecError> {
let input_stem = input
.file_stem()
.and_then(|stem| stem.to_str())
.ok_or_else(|| CodecError::NonUtf8Path(input.to_path_buf()))?;
let mut normalized = Vec::with_capacity(outputs.len());
for output in outputs {
let ext = output
.extension()
.and_then(|ext| ext.to_str())
.ok_or_else(|| CodecError::NonUtf8Path(output.clone()))?;
let target = output.with_file_name(format!("{input_stem}.{ext}"));
if output != target {
std::fs::rename(&output, &target).map_err(|source| CodecError::Io {
path: target.clone(),
source,
})?;
normalized.push(target);
} else {
normalized.push(output);
}
}
Ok(normalized)
}
assets-updater/src/core/config.rs
+207
View File
@@ -0,0 +1,207 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::core::errors::ConfigError;
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct AppConfig {
pub execution: ExecutionConfig,
pub tools: ToolsConfig,
pub concurrency: ConcurrencyConfig,
pub regions: BTreeMap<String, RegionConfig>,
}
impl AppConfig {
pub fn validate(&self) -> Result<(), ConfigError> {
for region_name in self.regions.keys() {
if region_name.to_lowercase() != *region_name {
return Err(ConfigError::InvalidRegionName(region_name.clone()));
}
}
Ok(())
}
pub fn enabled_regions(&self) -> Vec<String> {
self.regions
.iter()
.filter_map(|(name, region)| region.enabled.then_some(name.clone()))
.collect()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct AccessLogConfig {
pub enabled: bool,
pub format: String,
pub file: Option<String>,
}
impl Default for AccessLogConfig {
fn default() -> Self {
Self {
enabled: true,
format: "[${time}] ${status} - ${method} ${path} ${latency}\n".to_string(),
file: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct ToolsConfig {
pub ffmpeg_path: String,
pub asset_studio_cli_path: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct ExecutionConfig {
pub proxy: Option<String>,
pub timeout_seconds: u64,
pub allow_cancel: bool,
/// How many successful downloads to accumulate before flushing the download
/// record to disk mid-run. Set to `0` to disable mid-run flushing (record
/// is only written once at the end). Mirrors Go's `batchSaveSize`.
pub batch_save_size: usize,
pub retry: RetryConfig,
}
impl Default for ExecutionConfig {
fn default() -> Self {
Self {
proxy: None,
timeout_seconds: 300,
allow_cancel: true,
batch_save_size: 50,
retry: RetryConfig::default(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct RetryConfig {
pub attempts: usize,
pub initial_backoff_ms: u64,
pub max_backoff_ms: u64,
}
impl Default for RetryConfig {
fn default() -> Self {
Self {
attempts: 4,
initial_backoff_ms: 1_000,
max_backoff_ms: 4_000,
}
}
}
impl Default for ToolsConfig {
fn default() -> Self {
Self {
ffmpeg_path: "ffmpeg".to_string(),
asset_studio_cli_path: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct ConcurrencyConfig {
pub download: usize,
pub upload: usize,
pub acb: usize,
pub usm: usize,
pub hca: usize,
}
impl Default for ConcurrencyConfig {
fn default() -> Self {
Self {
download: 4,
upload: 4,
acb: 8,
usm: 4,
hca: 16,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
pub struct RegionConfig {
pub enabled: bool,
pub provider: RegionProviderConfig,
pub crypto: CryptoConfig,
pub runtime: RegionRuntimeConfig,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum RegionProviderConfig {
ColorfulPalette {
asset_info_url_template: String,
asset_bundle_url_template: String,
profile: String,
profile_hashes: BTreeMap<String, String>,
#[serde(default)]
required_cookies: bool,
#[serde(default)]
cookie_bootstrap_url: Option<String>,
},
Nuverse {
asset_version_url: String,
app_version: String,
asset_info_url_template: String,
asset_bundle_url_template: String,
#[serde(default)]
required_cookies: bool,
#[serde(default)]
cookie_bootstrap_url: Option<String>,
},
}
impl Default for RegionProviderConfig {
fn default() -> Self {
Self::ColorfulPalette {
asset_info_url_template: String::new(),
asset_bundle_url_template: String::new(),
profile: String::new(),
profile_hashes: BTreeMap::new(),
required_cookies: false,
cookie_bootstrap_url: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
pub struct CryptoConfig {
pub aes_key_hex: Option<String>,
pub aes_iv_hex: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct RegionRuntimeConfig {
pub unity_version: String,
}
impl Default for RegionRuntimeConfig {
fn default() -> Self {
Self {
unity_version: "2022.3.21f1".to_string(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
pub struct RegionPathsConfig {
pub asset_save_dir: Option<String>,
pub downloaded_asset_record_file: Option<String>,
}
assets-updater/src/core/errors.rs
+142
View File
@@ -0,0 +1,142 @@
use std::path::PathBuf;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum ConfigError {
#[error("failed to read config file {path}: {source}")]
Read {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("failed to parse config file {path}: {source}")]
Parse {
path: PathBuf,
#[source]
source: yaml_serde::Error,
},
#[error("config_version must be 2, got {0}")]
UnsupportedVersion(u32),
#[error("no v2 config file found; tried: {0}")]
MissingConfigFile(String),
#[error("invalid region key `{0}`; region keys must be lowercase")]
InvalidRegionName(String),
#[error("missing required environment variable `{name}` referenced by `{field}`")]
MissingEnvironmentVariable { field: String, name: String },
}
#[derive(Debug, Error)]
pub enum RegionError {
#[error("region `{0}` not found")]
NotFound(String),
#[error("region `{0}` is disabled")]
Disabled(String),
}
#[derive(Debug, Error)]
pub enum CodecError {
#[error("path {0} is not valid UTF-8 for cridecoder file APIs")]
NonUtf8Path(PathBuf),
#[error("io error at {path}: {source}")]
Io {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("ACB extraction failed: {0}")]
Acb(String),
#[error("USM extraction failed: {0}")]
Usm(String),
#[error("USM metadata read failed: {0}")]
Metadata(String),
#[error("HCA decode failed: {0}")]
Hca(String),
}
#[derive(Debug, Error)]
pub enum ExportPipelineError {
#[error(transparent)]
Codec(#[from] CodecError),
#[error("io error at {path}: {source}")]
Io {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("image codec error at {path}: {source}")]
Image {
path: PathBuf,
#[source]
source: image::ImageError,
},
#[error("failed to spawn command `{program}`: {source}")]
Spawn {
program: String,
#[source]
source: std::io::Error,
},
#[error("command `{program}` failed with status {status}: {stderr}")]
CommandFailed {
program: String,
status: String,
stderr: String,
},
#[error("failed to spawn worker `{worker}`: {source}")]
WorkerSpawn {
worker: String,
#[source]
source: std::io::Error,
},
#[error("worker `{worker}` panicked: {message}")]
WorkerPanic { worker: String, message: String },
}
#[derive(Debug, Error)]
pub enum AssetExecutionError {
#[error(transparent)]
Region(#[from] RegionError),
#[error(transparent)]
ExportPipeline(#[from] ExportPipelineError),
#[error("http request failed: {0}")]
Http(#[from] reqwest::Error),
#[error("failed to initialize HTTP client: {0}")]
HttpClient(String),
#[error("HTTP request to {url} returned status {status}")]
HttpStatus { url: String, status: u16 },
#[error("region `{region}` is missing asset_save_dir")]
MissingAssetSaveDir { region: String },
#[error("colorful_palette region `{region}` requires asset_version and asset_hash")]
MissingAssetVersionOrHash { region: String },
#[error("colorful_palette region `{region}` is missing profile hash for `{profile}`")]
MissingProfileHash { region: String, profile: String },
#[error("region `{region}` is missing AES key or IV configuration")]
MissingCryptoConfig { region: String },
#[error("invalid AES key hex: {0}")]
InvalidAesKeyHex(String),
#[error("invalid AES IV hex: {0}")]
InvalidAesIvHex(String),
#[error("invalid AES IV length: got {got}, want 16")]
InvalidAesIvLength { got: usize },
#[error("encrypted content cannot be empty")]
EmptyEncryptedContent,
#[error("encrypted content length is not a multiple of AES block size")]
InvalidEncryptedBlockSize,
#[error("failed to decrypt or deserialize asset info: {0}")]
AssetInfoDecode(String),
#[error("failed to create temp directory for {path}: {source}")]
CreateTempDir {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("failed to write temp file {path}: {source}")]
WriteTempFile {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("job execution cancelled")]
Cancelled,
}
assets-updater/src/core/export_pipeline.rs
+790
View File
@@ -0,0 +1,790 @@
use std::collections::VecDeque;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use crate::core::codec;
use crate::core::config::{AppConfig, RegionConfig};
use crate::core::errors::ExportPipelineError;
use crate::core::media::{
FrameRate, convert_m2v_to_mp4, convert_usm_to_mp4, convert_wav_to_flac, convert_wav_to_mp3,
};
use crate::core::retry::retry_async;
use common::updater::SyncContext;
use image::codecs::webp::WebPEncoder;
use image::{ExtendedColorType, ImageReader};
use log::debug;
use serde::Serialize;
use tokio::process::Command;
#[derive(Debug, Clone, Copy)]
struct AssetStudioCliCapabilities {
filter_exclude_mode: bool,
filter_blacklist_mode: bool,
sekai_keep_single_container_filename: bool,
}
#[derive(Debug, Clone, Serialize, Default)]
pub struct PostProcessSummary {
pub export_root: PathBuf,
pub generated_files: Vec<PathBuf>,
}
pub fn get_export_group(export_path: &str) -> &'static str {
if export_path.is_empty() {
return "container";
}
let normalized = export_path
.replace('\\', "/")
.trim_start_matches('/')
.to_lowercase();
for prefix in [
"event/center",
"event/thumbnail",
"gacha/icon",
"fix_prefab/mc_new",
"mysekai/character/",
] {
if normalized.starts_with(prefix) {
return "containerFull";
}
}
"container"
}
pub async fn extract_unity_asset_bundle(
app_config: &AppConfig,
sync_context: &SyncContext,
region: &RegionConfig,
asset_bundle_file: &Path,
export_path: &str,
category: &str,
) -> Result<PathBuf, ExportPipelineError> {
let output_dir = std::env::temp_dir()
.join("sekai-updater")
.join("extract")
.join(&sync_context.region);
let Some(asset_studio_cli_path) = app_config.tools.asset_studio_cli_path.as_deref() else {
return Ok(asset_bundle_file.parent().unwrap().to_path_buf());
};
let exclude_path_prefix = if sync_context.export.by_category {
"assets/sekai/assetbundle/resources".to_string()
} else if export_path.starts_with("mysekai") {
"assets/sekai/assetbundle/resources/ondemand".to_string()
} else {
format!(
"assets/sekai/assetbundle/resources/{}",
category.to_lowercase()
)
};
let actual_export_path = if sync_context.export.by_category {
output_dir.join(category.to_lowercase()).join(export_path)
} else {
output_dir.join(export_path)
};
debug!("{}", actual_export_path.to_string_lossy());
let capabilities = detect_assetstudio_cli_capabilities(asset_studio_cli_path);
let args = build_assetstudio_export_args(
asset_bundle_file,
output_dir.as_path(),
export_path,
&exclude_path_prefix,
region,
sync_context,
capabilities,
);
retry_async(
&app_config.execution.retry,
"assetstudio export",
|_| async {
let output = Command::new(asset_studio_cli_path)
.args(&args)
.output()
.await
.map_err(|source| ExportPipelineError::Spawn {
program: asset_studio_cli_path.to_string(),
source,
})?;
if output.status.success() {
Ok(())
} else {
Err(ExportPipelineError::CommandFailed {
program: asset_studio_cli_path.to_string(),
status: output.status.to_string(),
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
})
}
},
is_retryable_command_error,
)
.await?;
post_process_exported_files(app_config, sync_context, region, &actual_export_path).await?;
Ok(actual_export_path)
}
pub async fn post_process_exported_files(
app_config: &AppConfig,
sync_context: &SyncContext,
region: &RegionConfig,
export_path: &Path,
) -> Result<(), ExportPipelineError> {
if !export_path.exists() {
return Ok(());
}
handle_usm_files(
export_path,
sync_context,
region,
&app_config.tools.ffmpeg_path,
&app_config.execution.retry,
)
.await?;
handle_acb_files(
export_path,
sync_context,
region,
&app_config.tools.ffmpeg_path,
&app_config.execution.retry,
app_config.concurrency.acb,
app_config.concurrency.hca,
)
.await?;
handle_png_conversion(export_path, sync_context, region).await?;
Ok(())
}
async fn handle_usm_files(
export_path: &Path,
sync_context: &SyncContext,
region: &RegionConfig,
ffmpeg_path: &str,
retry: &crate::core::config::RetryConfig,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let usm_files = find_files_by_extension(export_path, "usm")?;
if !sync_context.export.usm.export || !sync_context.export.usm.decode || usm_files.is_empty() {
return Ok(Vec::new());
}
let usm_input = if usm_files.len() == 1 {
usm_files[0].clone()
} else {
merge_usm_files(export_path, &usm_files)?
};
process_usm_file(
&usm_input,
export_path,
sync_context,
region,
ffmpeg_path,
retry,
)
.await
}
async fn process_usm_file(
usm_file: &Path,
export_path: &Path,
sync_context: &SyncContext,
_: &RegionConfig,
ffmpeg_path: &str,
retry: &crate::core::config::RetryConfig,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let output_name = usm_file
.file_stem()
.and_then(|stem| stem.to_str())
.ok_or_else(|| ExportPipelineError::Io {
path: usm_file.to_path_buf(),
source: std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid usm file name"),
})?
.to_string();
if sync_context.export.video.convert_to_mp4
&& sync_context.export.video.direct_usm_to_mp4_with_ffmpeg
{
let mp4 = export_path.join(format!("{output_name}.mp4"));
convert_usm_to_mp4(usm_file, &mp4, ffmpeg_path, retry).await?;
remove_file_if_exists(usm_file)?;
return Ok(vec![mp4]);
}
let metadata = codec::read_usm_metadata(usm_file).ok();
let frame_rate = metadata
.as_ref()
.and_then(|metadata| metadata.video_frame_rate())
.filter(|(_, denominator)| *denominator > 0)
.map(FrameRate::from_tuple);
let extracted = codec::export_usm(usm_file, export_path)?;
let mut generated = extracted.clone();
if sync_context.export.video.convert_to_mp4 {
for extracted_file in extracted {
if extracted_file
.extension()
.and_then(|ext| ext.to_str())
.map(|ext| ext.eq_ignore_ascii_case("m2v"))
.unwrap_or(false)
{
let mp4 = export_path.join(format!("{output_name}.mp4"));
convert_m2v_to_mp4(
&extracted_file,
&mp4,
sync_context.export.video.remove_m2v,
ffmpeg_path,
frame_rate,
retry,
)
.await?;
generated.push(mp4);
if sync_context.export.video.remove_m2v {
generated.retain(|path| path != &extracted_file);
}
}
}
}
remove_file_if_exists(usm_file)?;
Ok(generated)
}
async fn handle_acb_files(
export_path: &Path,
sync_context: &SyncContext,
region: &RegionConfig,
ffmpeg_path: &str,
retry: &crate::core::config::RetryConfig,
acb_concurrency: usize,
hca_concurrency: usize,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let acb_files = find_files_by_extension(export_path, "acb")?;
if !sync_context.export.acb.export || !sync_context.export.acb.decode || acb_files.is_empty() {
return Ok(Vec::new());
}
let export_path = export_path.to_path_buf();
let region = region.clone();
let sync_context = sync_context.clone();
let ffmpeg_path = ffmpeg_path.to_string();
let retry = retry.clone();
run_path_tasks(acb_files, acb_concurrency, move |acb_file| {
process_acb_file(
&acb_file,
&export_path,
&sync_context,
&region,
&ffmpeg_path,
&retry,
hca_concurrency,
)
})
}
fn process_acb_file(
acb_file: &Path,
output_dir: &Path,
sync_context: &SyncContext,
region: &RegionConfig,
ffmpeg_path: &str,
retry: &crate::core::config::RetryConfig,
hca_concurrency: usize,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let parent_dir = acb_file.parent().ok_or_else(|| ExportPipelineError::Io {
path: acb_file.to_path_buf(),
source: std::io::Error::new(std::io::ErrorKind::NotFound, "missing parent directory"),
})?;
let extract_dir = tempfile::Builder::new()
.prefix("acb-extract-")
.tempdir_in(parent_dir)
.map_err(|source| ExportPipelineError::Io {
path: parent_dir.to_path_buf(),
source,
})?;
let _ = codec::export_acb(acb_file, extract_dir.path())?;
let mut hca_files = find_files_by_extension(extract_dir.path(), "hca")?;
let acb_path_lower = acb_file.to_string_lossy().replace('\\', "/").to_lowercase();
if acb_path_lower.contains("music/long") {
hca_files.retain(|path| {
let lower = path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_default()
.to_lowercase();
!(lower.ends_with("_vr.hca") || lower.ends_with("_screen.hca"))
});
}
if !sync_context.export.hca.decode {
remove_file_if_exists(acb_file)?;
return Ok(Vec::new());
}
let extract_output_dir = extract_dir.path().to_path_buf();
let region = region.clone();
let sync_context = sync_context.clone();
let ffmpeg_path = ffmpeg_path.to_string();
let retry = retry.clone();
let generated = run_path_tasks(hca_files, hca_concurrency, move |hca_file| {
process_hca_file(
&hca_file,
&extract_output_dir,
&sync_context,
&region,
&ffmpeg_path,
&retry,
)
})?;
let final_outputs = move_result_files(output_dir, &generated)?;
remove_file_if_exists(acb_file)?;
Ok(final_outputs)
}
fn process_hca_file(
hca_file: &Path,
output_dir: &Path,
sync_context: &SyncContext,
_: &RegionConfig,
ffmpeg_path: &str,
retry: &crate::core::config::RetryConfig,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let base_name = hca_file
.file_stem()
.and_then(|stem| stem.to_str())
.ok_or_else(|| ExportPipelineError::Io {
path: hca_file.to_path_buf(),
source: std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid hca file name"),
})?;
let wav_file = hca_file.with_extension("wav");
codec::decode_hca_to_wav(hca_file, &wav_file)?;
remove_file_if_exists(hca_file)?;
let mut generated = vec![wav_file.clone()];
if sync_context.export.audio.convert_to_mp3 {
let mp3 = output_dir.join(format!("{base_name}.mp3"));
convert_wav_to_mp3(&wav_file, &mp3, ffmpeg_path, retry)?;
if sync_context.export.audio.remove_wav {
remove_file_if_exists(&wav_file)?;
generated.retain(|path| path != &wav_file);
}
generated.push(mp3);
} else if sync_context.export.audio.convert_to_flac {
let flac = output_dir.join(format!("{base_name}.flac"));
convert_wav_to_flac(&wav_file, &flac, ffmpeg_path, retry)?;
if sync_context.export.audio.remove_wav {
remove_file_if_exists(&wav_file)?;
generated.retain(|path| path != &wav_file);
}
generated.push(flac);
} else if sync_context.export.audio.remove_wav {
remove_file_if_exists(&wav_file)?;
generated.clear();
}
let final_outputs = move_result_files(output_dir, &generated)?;
Ok(final_outputs)
}
async fn handle_png_conversion(
export_path: &Path,
sync_context: &SyncContext,
_: &RegionConfig,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
if !sync_context.export.images.convert_to_webp {
return Ok(Vec::new());
}
let png_files = find_files_by_extension(export_path, "png")?;
let mut generated = Vec::new();
for png_file in png_files {
let webp = png_file.with_extension("webp");
convert_png_to_webp(&png_file, &webp)?;
generated.push(webp.clone());
if sync_context.export.images.remove_png {
remove_file_if_exists(&png_file)?;
}
}
Ok(generated)
}
fn convert_png_to_webp(png_file: &Path, webp_file: &Path) -> Result<(), ExportPipelineError> {
let image = ImageReader::open(png_file)
.map_err(|source| ExportPipelineError::Io {
path: png_file.to_path_buf(),
source,
})?
.decode()
.map_err(|source| ExportPipelineError::Image {
path: png_file.to_path_buf(),
source,
})?;
let rgba = image.to_rgba8();
let (width, height) = rgba.dimensions();
let writer = std::fs::File::create(webp_file).map_err(|source| ExportPipelineError::Io {
path: webp_file.to_path_buf(),
source,
})?;
let writer = std::io::BufWriter::new(writer);
WebPEncoder::new_lossless(writer)
.encode(rgba.as_raw(), width, height, ExtendedColorType::Rgba8)
.map_err(|source| ExportPipelineError::Image {
path: webp_file.to_path_buf(),
source,
})
}
fn is_retryable_command_error(err: &ExportPipelineError) -> bool {
match err {
ExportPipelineError::Spawn { source, .. } => matches!(
source.kind(),
std::io::ErrorKind::Interrupted
| std::io::ErrorKind::TimedOut
| std::io::ErrorKind::WouldBlock
| std::io::ErrorKind::BrokenPipe
| std::io::ErrorKind::ConnectionReset
| std::io::ErrorKind::ConnectionAborted
| std::io::ErrorKind::ConnectionRefused
),
ExportPipelineError::CommandFailed { .. } => true,
_ => false,
}
}
fn build_assetstudio_export_args(
asset_bundle_file: &Path,
output_dir: &Path,
export_path: &str,
exclude_path_prefix: &str,
region: &RegionConfig,
sync_context: &SyncContext,
capabilities: AssetStudioCliCapabilities,
) -> Vec<String> {
let mut args = vec![
asset_bundle_file.to_string_lossy().to_string(),
"-m".to_string(),
"export".to_string(),
"-t".to_string(),
"monoBehaviour,textAsset,tex2d,tex2dArray,audio".to_string(),
"-g".to_string(),
get_export_group(export_path).to_string(),
"-f".to_string(),
"assetName".to_string(),
"-o".to_string(),
output_dir.to_string_lossy().to_string(),
"--strip-path-prefix".to_string(),
exclude_path_prefix.to_string(),
"-r".to_string(),
];
if capabilities.filter_exclude_mode {
args.push("--filter-exclude-mode".to_string());
} else if capabilities.filter_blacklist_mode {
args.push("--filter-blacklist-mode".to_string());
}
args.push("--filter-with-regex".to_string());
if capabilities.sekai_keep_single_container_filename {
args.push("--sekai-keep-single-container-filename".to_string());
}
if !region.runtime.unity_version.is_empty() {
args.push("--unity-version".to_string());
args.push(region.runtime.unity_version.clone());
}
let mut excluded_exts = Vec::new();
if !sync_context.export.usm.export {
excluded_exts.push("usm");
}
if !sync_context.export.acb.export {
excluded_exts.push("acb");
}
if !excluded_exts.is_empty() {
args.push("--filter-by-name".to_string());
args.push(format!(r".*\.({})$", excluded_exts.join("|")));
}
args
}
fn detect_assetstudio_cli_capabilities(asset_studio_cli_path: &str) -> AssetStudioCliCapabilities {
static CACHE: std::sync::OnceLock<
Mutex<std::collections::HashMap<String, AssetStudioCliCapabilities>>,
> = std::sync::OnceLock::new();
let cache = CACHE.get_or_init(|| Mutex::new(std::collections::HashMap::new()));
if let Some(cached) = cache.lock().unwrap().get(asset_studio_cli_path).copied() {
return cached;
}
let fallback = AssetStudioCliCapabilities {
filter_exclude_mode: true,
filter_blacklist_mode: false,
sekai_keep_single_container_filename: true,
};
let detected = match std::process::Command::new(asset_studio_cli_path)
.arg("--help")
.output()
{
Ok(output) => {
let help = format!(
"{}\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
AssetStudioCliCapabilities {
filter_exclude_mode: help.contains("--filter-exclude-mode"),
filter_blacklist_mode: help.contains("--filter-blacklist-mode"),
sekai_keep_single_container_filename: help
.contains("--sekai-keep-single-container-filename"),
}
}
Err(_) => fallback,
};
cache
.lock()
.unwrap()
.insert(asset_studio_cli_path.to_string(), detected);
detected
}
fn move_result_files(
output_dir: &Path,
generated: &[PathBuf],
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let mut final_outputs = Vec::new();
for path in generated {
let file_name = match path.file_name() {
Some(name) => name,
None => continue,
};
let destination = output_dir.join(file_name);
if path != &destination {
std::fs::rename(path, &destination).map_err(|source| ExportPipelineError::Io {
path: destination.clone(),
source,
})?;
final_outputs.push(destination);
} else if destination.exists() {
final_outputs.push(destination);
}
}
Ok(final_outputs)
}
fn run_path_tasks<F>(
paths: Vec<PathBuf>,
concurrency: usize,
task: F,
) -> Result<Vec<PathBuf>, ExportPipelineError>
where
F: Fn(PathBuf) -> Result<Vec<PathBuf>, ExportPipelineError> + Send + Sync + 'static,
{
if paths.is_empty() {
return Ok(Vec::new());
}
let worker_count = concurrency.max(1).min(paths.len());
let queue = Arc::new(Mutex::new(VecDeque::from(paths)));
let results = Arc::new(Mutex::new(Vec::<PathBuf>::new()));
let first_error = Arc::new(Mutex::new(None::<ExportPipelineError>));
let task = Arc::new(task);
let mut handles = Vec::with_capacity(worker_count);
const WORKER_STACK_SIZE: usize = 32 * 1024 * 1024;
for _ in 0..worker_count {
let queue = queue.clone();
let results = results.clone();
let first_error = first_error.clone();
let task = task.clone();
let worker_name = "export-task".to_string();
let handle = std::thread::Builder::new()
.name(worker_name.clone())
.stack_size(WORKER_STACK_SIZE)
.spawn(move || {
loop {
if first_error.lock().unwrap().is_some() {
break;
}
let next_path = queue.lock().unwrap().pop_front();
let Some(path) = next_path else {
break;
};
match task(path) {
Ok(mut generated) => results.lock().unwrap().append(&mut generated),
Err(err) => {
let mut first = first_error.lock().unwrap();
if first.is_none() {
*first = Some(err);
}
break;
}
}
}
})
.map_err(|source| ExportPipelineError::WorkerSpawn {
worker: worker_name,
source,
})?;
handles.push(handle);
}
for handle in handles {
handle
.join()
.map_err(|panic| ExportPipelineError::WorkerPanic {
worker: "export task".to_string(),
message: panic_message(panic),
})?;
}
if let Some(err) = first_error.lock().unwrap().take() {
return Err(err);
}
let mut results = results.lock().unwrap();
Ok(std::mem::take(&mut *results))
}
fn panic_message(panic: Box<dyn std::any::Any + Send>) -> String {
if let Some(message) = panic.downcast_ref::<&str>() {
(*message).to_string()
} else if let Some(message) = panic.downcast_ref::<String>() {
message.clone()
} else {
"unknown worker panic".to_string()
}
}
fn merge_usm_files(dir: &Path, usm_files: &[PathBuf]) -> Result<PathBuf, ExportPipelineError> {
let dir_name = dir
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("merged");
let merged_file = dir.join(format!("{dir_name}.usm"));
let mut target =
std::fs::File::create(&merged_file).map_err(|source| ExportPipelineError::Io {
path: merged_file.clone(),
source,
})?;
for source_path in usm_files {
if *source_path == merged_file {
continue;
}
let mut source =
std::fs::File::open(source_path).map_err(|source| ExportPipelineError::Io {
path: source_path.clone(),
source,
})?;
std::io::copy(&mut source, &mut target).map_err(|source| ExportPipelineError::Io {
path: source_path.clone(),
source,
})?;
drop(source);
remove_file_if_exists(source_path)?;
}
Ok(merged_file)
}
pub fn find_files(dir: &Path) -> Result<Vec<PathBuf>, ExportPipelineError> {
let mut files = Vec::new();
walk(dir, &mut |path| {
files.push(path.to_path_buf());
})?;
Ok(files)
}
pub fn find_files_by_extension(dir: &Path, ext: &str) -> Result<Vec<PathBuf>, ExportPipelineError> {
let target_ext = ext.to_lowercase();
let mut files = Vec::new();
walk(dir, &mut |path| {
if path
.extension()
.and_then(|value| value.to_str())
.map(|value| value.eq_ignore_ascii_case(&target_ext))
.unwrap_or(false)
{
files.push(path.to_path_buf());
}
})?;
Ok(files)
}
pub fn find_files_by_extensions<P>(
dir: &Path,
ext: &[P],
) -> Result<Vec<PathBuf>, ExportPipelineError>
where
P: AsRef<str>,
{
let target_ext = ext
.iter()
.map(|x| x.as_ref().to_lowercase())
.collect::<Vec<_>>();
let mut files = Vec::new();
walk(dir, &mut |path| {
if path
.extension()
.and_then(|value| value.to_str())
.map(|value| target_ext.contains(&value.to_lowercase()))
.unwrap_or(false)
{
files.push(path.to_path_buf());
}
})?;
Ok(files)
}
fn walk(dir: &Path, f: &mut dyn FnMut(&Path)) -> Result<(), ExportPipelineError> {
for entry in std::fs::read_dir(dir).map_err(|source| ExportPipelineError::Io {
path: dir.to_path_buf(),
source,
})? {
let entry = entry.map_err(|source| ExportPipelineError::Io {
path: dir.to_path_buf(),
source,
})?;
let path = entry.path();
let file_type = entry
.file_type()
.map_err(|source| ExportPipelineError::Io {
path: path.clone(),
source,
})?;
if file_type.is_dir() {
walk(&path, f)?;
} else {
f(&path);
}
}
Ok(())
}
fn remove_file_if_exists(path: &Path) -> Result<(), ExportPipelineError> {
if path.exists() {
std::fs::remove_file(path).map_err(|source| ExportPipelineError::Io {
path: path.to_path_buf(),
source,
})?;
}
Ok(())
}
assets-updater/src/core/media.rs
+229
View File
@@ -0,0 +1,229 @@
use std::fmt::{Display, Formatter};
use std::path::Path;
use tokio::process::Command;
use crate::core::config::RetryConfig;
use crate::core::errors::ExportPipelineError;
use crate::core::retry::{retry_async, retry_sync};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct FrameRate {
pub numerator: i32,
pub denominator: i32,
}
impl FrameRate {
pub fn from_tuple((numerator, denominator): (i32, i32)) -> Self {
Self {
numerator,
denominator,
}
}
}
impl Display for FrameRate {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
if self.denominator <= 1 {
write!(f, "{}", self.numerator)
} else {
write!(f, "{}/{}", self.numerator, self.denominator)
}
}
}
pub async fn convert_usm_to_mp4(
usm_file: &Path,
mp4_file: &Path,
ffmpeg_path: &str,
retry: &RetryConfig,
) -> Result<(), ExportPipelineError> {
retry_async(
retry,
"ffmpeg usm->mp4",
|_| async {
run_ffmpeg(
ffmpeg_path,
&[
"-i",
&usm_file.to_string_lossy(),
"-c:v",
"libx264",
"-c:a",
"aac",
"-b:a",
"192k",
"-movflags",
"+faststart",
"-y",
&mp4_file.to_string_lossy(),
],
)
.run_async()
.await
},
is_retryable_command_error,
)
.await
}
pub async fn convert_m2v_to_mp4(
m2v_file: &Path,
mp4_file: &Path,
delete_original: bool,
ffmpeg_path: &str,
frame_rate: Option<FrameRate>,
retry: &RetryConfig,
) -> Result<(), ExportPipelineError> {
let mut args = Vec::new();
if let Some(rate) = frame_rate {
args.push("-r".to_string());
args.push(rate.to_string());
}
args.push("-i".to_string());
args.push(m2v_file.to_string_lossy().to_string());
args.push("-c:v".to_string());
args.push("libx264".to_string());
if let Some(rate) = frame_rate {
args.push("-r".to_string());
args.push(rate.to_string());
}
args.push("-y".to_string());
args.push(mp4_file.to_string_lossy().to_string());
retry_async(
retry,
"ffmpeg m2v->mp4",
|_| async {
let refs: Vec<&str> = args.iter().map(String::as_str).collect();
run_ffmpeg(ffmpeg_path, &refs).run_async().await
},
is_retryable_command_error,
)
.await?;
if delete_original && m2v_file.exists() {
std::fs::remove_file(m2v_file).map_err(|source| ExportPipelineError::Io {
path: m2v_file.to_path_buf(),
source,
})?;
}
Ok(())
}
pub fn convert_wav_to_mp3(
wav_file: &Path,
mp3_file: &Path,
ffmpeg_path: &str,
retry: &RetryConfig,
) -> Result<(), ExportPipelineError> {
retry_sync(
retry,
"ffmpeg wav->mp3",
|_| {
run_ffmpeg_sync(
ffmpeg_path,
&[
"-i",
&wav_file.to_string_lossy(),
"-b:a",
"320k",
"-y",
&mp3_file.to_string_lossy(),
],
)
},
is_retryable_command_error,
)
}
pub fn convert_wav_to_flac(
wav_file: &Path,
flac_file: &Path,
ffmpeg_path: &str,
retry: &RetryConfig,
) -> Result<(), ExportPipelineError> {
retry_sync(
retry,
"ffmpeg wav->flac",
|_| {
run_ffmpeg_sync(
ffmpeg_path,
&[
"-i",
&wav_file.to_string_lossy(),
"-compression_level",
"12",
"-y",
&flac_file.to_string_lossy(),
],
)
},
is_retryable_command_error,
)
}
fn run_ffmpeg<'a>(ffmpeg_path: &'a str, args: &'a [&'a str]) -> FfmpegCommand<'a> {
FfmpegCommand { ffmpeg_path, args }
}
fn run_ffmpeg_sync(ffmpeg_path: &str, args: &[&str]) -> Result<(), ExportPipelineError> {
let output = std::process::Command::new(ffmpeg_path)
.args(args)
.output()
.map_err(|source| ExportPipelineError::Spawn {
program: ffmpeg_path.to_string(),
source,
})?;
map_command_output(ffmpeg_path, output)
}
struct FfmpegCommand<'a> {
ffmpeg_path: &'a str,
args: &'a [&'a str],
}
impl<'a> FfmpegCommand<'a> {
async fn run_async(self) -> Result<(), ExportPipelineError> {
let output = Command::new(self.ffmpeg_path)
.args(self.args)
.output()
.await
.map_err(|source| ExportPipelineError::Spawn {
program: self.ffmpeg_path.to_string(),
source,
})?;
map_command_output(self.ffmpeg_path, output)
}
}
fn is_retryable_command_error(err: &ExportPipelineError) -> bool {
match err {
ExportPipelineError::Spawn { source, .. } => matches!(
source.kind(),
std::io::ErrorKind::Interrupted
| std::io::ErrorKind::TimedOut
| std::io::ErrorKind::WouldBlock
| std::io::ErrorKind::BrokenPipe
| std::io::ErrorKind::ConnectionReset
| std::io::ErrorKind::ConnectionAborted
| std::io::ErrorKind::ConnectionRefused
),
ExportPipelineError::CommandFailed { .. } => true,
_ => false,
}
}
fn map_command_output(
program: &str,
output: std::process::Output,
) -> Result<(), ExportPipelineError> {
if output.status.success() {
Ok(())
} else {
Err(ExportPipelineError::CommandFailed {
program: program.to_string(),
status: output.status.to_string(),
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
})
}
}
assets-updater/src/core/mod.rs
+8
View File
@@ -0,0 +1,8 @@
pub mod asset_execution;
pub mod config;
pub mod export_pipeline;
pub mod regions;
pub mod retry;
pub mod media;
pub mod codec;
pub mod errors;
assets-updater/src/core/regions.rs
+30
View File
@@ -0,0 +1,30 @@
use regex::Regex;
use crate::core::config::{AppConfig, RegionConfig};
use crate::core::errors::RegionError;
pub fn select_region<'a>(
config: &'a AppConfig,
name: &str,
) -> Result<&'a RegionConfig, RegionError> {
let region = config
.regions
.get(name)
.ok_or_else(|| RegionError::NotFound(name.to_string()))?;
if !region.enabled {
return Err(RegionError::Disabled(name.to_string()));
}
Ok(region)
}
pub(crate) fn compile_patterns(patterns: &[String]) -> Vec<Regex> {
patterns
.iter()
.filter_map(|pattern| Regex::new(pattern).ok())
.collect()
}
pub(crate) fn matches_any(patterns: &[Regex], bundle_name: &str) -> bool {
patterns.iter().any(|regex| regex.is_match(bundle_name))
}
assets-updater/src/core/retry.rs
+83
View File
@@ -0,0 +1,83 @@
use std::fmt::Display;
use std::future::Future;
use std::time::Duration;
use log::warn;
use tokio::time::sleep;
use crate::core::config::RetryConfig;
pub async fn retry_async<T, E, Op, Fut, ShouldRetry>(
config: &RetryConfig,
operation: &str,
mut op: Op,
should_retry: ShouldRetry,
) -> Result<T, E>
where
E: Display,
Op: FnMut(usize) -> Fut,
Fut: Future<Output = Result<T, E>>,
ShouldRetry: Fn(&E) -> bool,
{
let attempts = config.attempts.max(1);
for attempt in 1..=attempts {
match op(attempt).await {
Ok(value) => return Ok(value),
Err(err) if attempt < attempts && should_retry(&err) => {
let delay = backoff_delay(config, attempt);
warn!(
"operation '{}' failed after {} attempt(s) with error: {}, retrying in {} ms",
operation,
attempt,
err,
delay.as_millis()
);
sleep(delay).await;
}
Err(err) => return Err(err),
}
}
unreachable!("retry_async must return from within the attempt loop")
}
pub fn retry_sync<T, E, Op, ShouldRetry>(
config: &RetryConfig,
operation: &str,
mut op: Op,
should_retry: ShouldRetry,
) -> Result<T, E>
where
E: Display,
Op: FnMut(usize) -> Result<T, E>,
ShouldRetry: Fn(&E) -> bool,
{
let attempts = config.attempts.max(1);
for attempt in 1..=attempts {
match op(attempt) {
Ok(value) => return Ok(value),
Err(err) if attempt < attempts && should_retry(&err) => {
let delay = backoff_delay(config, attempt);
warn!(
"operation '{}' failed after {} attempt(s) with error: {}, retrying in {} ms",
operation,
attempt,
err,
delay.as_millis()
);
std::thread::sleep(delay);
}
Err(err) => return Err(err),
}
}
unreachable!("retry_sync must return from within the attempt loop")
}
fn backoff_delay(config: &RetryConfig, attempt: usize) -> Duration {
let base = config.initial_backoff_ms.max(1);
let max = config.max_backoff_ms.max(base);
let multiplier = 1u64
.checked_shl(attempt.saturating_sub(1) as u32)
.unwrap_or(u64::MAX);
Duration::from_millis(base.saturating_mul(multiplier).min(max))
}
assets-updater/src/lib.rs
+1
View File
@@ -0,0 +1 @@
pub mod core;