Bluemangoo/nonebot_sekai_update_notify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: Bluemangoo:master
Branches Tags
Bluemangoo:master
Bluemangoo:lib
...
pull from: Bluemangoo:lib
Branches Tags
Bluemangoo:master
Bluemangoo:lib

No commits in common. "master" and "lib" have entirely different histories.
master ... lib

29 changed files with 5676 additions and 204 deletions
Show Stats Expand all files Collapse all files

6
.gitignore vendored
View File

@ -1,4 +1,2 @@
__pycache__
*.dll
*.so
*.dylib
target/
.idea/

2935
Cargo.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

23
__init__.py
View File

@ -1,23 +0,0 @@
import nonebot
from nonebot import require
require("nonebot_plugin_apscheduler")
from pathlib import Path
from nonebot_plugin_apscheduler import scheduler
from .config import pconfig
from .lib import rlib
from .sync import sync
driver = nonebot.get_driver()
@driver.on_startup
def startup():
if pconfig.proxy:
rlib.set_proxy(pconfig.proxy)
if pconfig.github_token:
rlib.set_ghp(pconfig.github_token)
rlib.boot(str(Path.cwd()), "nonebot_plugin_sekai_update_notify", pconfig.assets_studio_path)
scheduler.add_job(sync, "interval", seconds=10, id="sekai_update_sync", replace_existing=True)

25
assets-updater/Cargo.toml Normal file
View File

@ -0,0 +1,25 @@
[package]
name = "assets-updater"
version = "0.1.0"
edition = "2024"
[dependencies]
common = { path = "../common" }
aes = { workspace = true }
tokio = { workspace = true, features = ["process"] }
rmp-serde = { workspace = true }
hex = { workspace = true }
cbc = { workspace = true }
tempfile = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
reqwest = { workspace = true }
serde = { workspace = true, features = ["derive"] }
regex = { workspace = true }
yaml_serde = { workspace = true }
image = { workspace = true }
cridecoder = { workspace = true }
thiserror = { workspace = true }
log = { workspace = true }
cipher = { workspace = true, features = ["block-padding"] }
twox-hash = { workspace = true, features = ["xxhash3_128"] }

384
assets-updater/src/core/asset_execution.rs Normal file
View File

@ -0,0 +1,384 @@
use std::collections::HashMap;
use std::path::PathBuf;
use std::time::Duration;
use crate::core::config::{AppConfig, RegionConfig, RegionProviderConfig};
use crate::core::errors::AssetExecutionError;
use crate::core::export_pipeline::extract_unity_asset_bundle;
use crate::core::retry::retry_async;
use aes::cipher::block_padding::Pkcs7;
use aes::cipher::{BlockDecryptMut, KeyIvInit};
use chrono::FixedOffset;
use common::updater::{AssetCategory, SyncContext};
use reqwest::header::{
ACCEPT, ACCEPT_ENCODING, ACCEPT_LANGUAGE, CONNECTION, COOKIE, HeaderMap, HeaderValue,
SET_COOKIE, USER_AGENT,
};
use serde::{Deserialize, Serialize};
type Aes128CbcDec = cbc::Decryptor<aes::Aes128>;
type Aes192CbcDec = cbc::Decryptor<aes::Aes192>;
type Aes256CbcDec = cbc::Decryptor<aes::Aes256>;
/// Deserializes a msgpack/JSON null or missing value as an empty String.
/// Go silently coerces nil → zero value for non-pointer types; this helper
/// mirrors that behavior for String fields.
fn de_null_as_empty_string<'de, D>(deserializer: D) -> Result<String, D::Error>
where
D: serde::Deserializer<'de>,
{
Ok(Option::<String>::deserialize(deserializer)?.unwrap_or_default())
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AssetBundleDetail {
#[serde(rename = "bundleName", deserialize_with = "de_null_as_empty_string")]
pub bundle_name: String,
#[serde(rename = "cacheFileName", deserialize_with = "de_null_as_empty_string")]
pub cache_file_name: String,
#[serde(
rename = "cacheDirectoryName",
deserialize_with = "de_null_as_empty_string"
)]
pub cache_directory_name: String,
// nuverse regions use `crc` instead of `hash`; the server may send nil here.
#[serde(rename = "hash", deserialize_with = "de_null_as_empty_string")]
pub hash: String,
#[serde(rename = "category")]
pub category: AssetCategory,
#[serde(rename = "crc")]
pub crc: i64,
#[serde(rename = "fileSize")]
pub file_size: i64,
#[serde(rename = "dependencies")]
pub dependencies: Vec<String>,
#[serde(rename = "paths", default)]
pub paths: Vec<String>,
#[serde(rename = "isBuiltin")]
pub is_builtin: bool,
#[serde(rename = "isRelocate")]
pub is_relocate: Option<bool>,
#[serde(rename = "md5Hash")]
pub md5_hash: Option<String>,
#[serde(rename = "downloadPath")]
pub download_path: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AssetBundleInfo {
#[serde(rename = "version")]
pub version: Option<String>,
#[serde(rename = "os")]
pub os: Option<String>,
#[serde(rename = "bundles")]
pub bundles: HashMap<String, AssetBundleDetail>,
}
#[derive(Debug, Clone)]
pub struct AssetExecutionContext {
client: reqwest::Client,
region_name: String,
region: RegionConfig,
retry: crate::core::config::RetryConfig,
runtime_cookie: Option<String>,
resolved_asset_version: Option<String>,
pub sync_context: SyncContext,
}
impl AssetExecutionContext {
pub fn new(
app_config: &AppConfig,
sync_context: &SyncContext,
region: &RegionConfig,
) -> Result<Self, AssetExecutionError> {
let mut headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static("*/*"));
headers.insert(
USER_AGENT,
HeaderValue::from_static("ProductName/134 CFNetwork/1408.0.4 Darwin/22.5.0"),
);
headers.insert(CONNECTION, HeaderValue::from_static("keep-alive"));
headers.insert(
ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br"),
);
headers.insert(
ACCEPT_LANGUAGE,
HeaderValue::from_static("zh-CN,zh-Hans;q=0.9"),
);
headers.insert(
"X-Unity-Version",
HeaderValue::from_str(&region.runtime.unity_version)
.map_err(|err| AssetExecutionError::HttpClient(err.to_string()))?,
);
let mut builder = reqwest::Client::builder()
.default_headers(headers)
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(180))
.pool_max_idle_per_host(100)
.tcp_keepalive(Duration::from_secs(30));
if let Some(proxy) = &app_config.execution.proxy
&& !proxy.is_empty()
{
builder = builder.proxy(
reqwest::Proxy::all(proxy)
.map_err(|err| AssetExecutionError::HttpClient(err.to_string()))?,
);
} else {
builder = builder.no_proxy();
}
Ok(Self {
client: builder
.build()
.map_err(|err| AssetExecutionError::HttpClient(err.to_string()))?,
sync_context: sync_context.clone(),
region_name: sync_context.region.clone(),
region: region.clone(),
retry: app_config.execution.retry.clone(),
runtime_cookie: None,
resolved_asset_version: sync_context.asset_version.clone(),
})
}
pub async fn download(
&self,
task: &str,
app_config: &AppConfig,
) -> Result<(PathBuf, /*single file*/ bool), AssetExecutionError> {
let ctx = self.clone();
ctx.download_and_export_bundle(app_config, task).await
}
fn render_bundle_url(&self, task: &str) -> Result<String, AssetExecutionError> {
match &self.region.provider {
RegionProviderConfig::ColorfulPalette {
asset_bundle_url_template,
profile_hash,
..
} => {
let asset_version =
self.sync_context.asset_version.as_deref().ok_or_else(|| {
AssetExecutionError::MissingAssetVersionOrHash {
region: self.region_name.clone(),
}
})?;
let asset_hash = self.sync_context.asset_hash.as_deref().ok_or_else(|| {
AssetExecutionError::MissingAssetVersionOrHash {
region: self.region_name.clone(),
}
})?;
Ok(asset_bundle_url_template
.replace("{bundle_path}", task)
.replace("{asset_version}", asset_version)
.replace("{asset_hash}", asset_hash)
.replace("{env}", "production")
.replace("{hash}", profile_hash)
+ &time_arg_jst())
}
RegionProviderConfig::Nuverse {
asset_bundle_url_template,
..
} => {
let app_version = self.sync_context.app_version.as_ref().ok_or_else(|| {
AssetExecutionError::MissingAppVersion {
region: self.region_name.clone(),
}
})?;
let asset_version = self
.resolved_asset_version
.as_deref()
.unwrap_or("<resolved-asset-version>");
Ok(asset_bundle_url_template
.replace("{bundle_path}", task)
.replace("{app_version}", app_version)
.replace("{asset_version}", asset_version)
+ &time_arg_jst())
}
}
}
async fn get_with_retry(&self, url: &str) -> Result<Vec<u8>, AssetExecutionError> {
retry_async(
&self.retry,
"http get",
|_| async {
let mut request = self.client.get(url);
if let Some(cookie) = &self.runtime_cookie {
request = request.header(COOKIE, cookie);
}
match request.send().await {
Ok(response) if response.status().is_success() => {
Ok(response.bytes().await?.to_vec())
}
Ok(response) => Err(AssetExecutionError::HttpStatus {
url: url.to_string(),
status: response.status().as_u16(),
}),
Err(err) => Err(AssetExecutionError::Http(err)),
}
},
is_retryable_http_error,
)
.await
}
pub async fn fetch_runtime_cookies(&mut self) -> Result<(), AssetExecutionError> {
let url = "https://issue.sekai.colorfulpalette.org/api/signature".to_string();
self.runtime_cookie = retry_async(
&self.retry,
"cookie bootstrap",
|_| async {
let response = self.client.post(&url).send().await?;
if response.status().is_success() {
Ok(response
.headers()
.get(SET_COOKIE)
.and_then(|value| value.to_str().ok())
.map(str::to_string))
} else {
Err(AssetExecutionError::HttpStatus {
url: url.clone(),
status: response.status().as_u16(),
})
}
},
is_retryable_http_error,
)
.await?;
Ok(())
}
async fn download_and_export_bundle(
&self,
app_config: &AppConfig,
task: &str,
) -> Result<(PathBuf, bool), AssetExecutionError> {
let bundle_url = self.render_bundle_url(task)?;
let body = self.get_with_retry(&bundle_url).await?;
let deobfuscated = deobfuscate(&body);
let temp_file = std::env::temp_dir()
.join("sekai-updater")
.join("obf")
.join(&self.region_name)
.join(task);
if let Some(parent) = temp_file.parent() {
std::fs::create_dir_all(parent).map_err(|source| {
AssetExecutionError::CreateTempDir {
path: parent.to_path_buf(),
source,
}
})?;
}
std::fs::write(&temp_file, deobfuscated).map_err(|source| {
AssetExecutionError::WriteTempFile {
path: temp_file.clone(),
source,
}
})?;
let export_result = extract_unity_asset_bundle(
app_config,
&self.sync_context,
&self.region,
&temp_file,
task,
task,
)
.await;
let _ = std::fs::remove_file(&temp_file);
export_result.map_err(Into::into)
}
}
fn is_retryable_http_error(err: &AssetExecutionError) -> bool {
match err {
AssetExecutionError::Http(_) => true,
AssetExecutionError::HttpStatus { status, .. } => *status >= 500,
_ => false,
}
}
pub fn decrypt_asset_bundle_info(
aes_key_hex: &str,
aes_iv_hex: &str,
content: &[u8],
) -> Result<AssetBundleInfo, AssetExecutionError> {
if content.is_empty() {
return Err(AssetExecutionError::EmptyEncryptedContent);
}
if !content.len().is_multiple_of(16) {
return Err(AssetExecutionError::InvalidEncryptedBlockSize);
}
let key = hex::decode(aes_key_hex)
.map_err(|err| AssetExecutionError::InvalidAesKeyHex(err.to_string()))?;
let iv = hex::decode(aes_iv_hex)
.map_err(|err| AssetExecutionError::InvalidAesIvHex(err.to_string()))?;
if iv.len() != 16 {
return Err(AssetExecutionError::InvalidAesIvLength { got: iv.len() });
}
let mut buf = content.to_vec();
let decrypted = match key.len() {
16 => Aes128CbcDec::new_from_slices(&key, &iv)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?
.decrypt_padded_mut::<Pkcs7>(&mut buf)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?,
24 => Aes192CbcDec::new_from_slices(&key, &iv)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?
.decrypt_padded_mut::<Pkcs7>(&mut buf)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?,
32 => Aes256CbcDec::new_from_slices(&key, &iv)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?
.decrypt_padded_mut::<Pkcs7>(&mut buf)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))?,
_ => {
return Err(AssetExecutionError::AssetInfoDecode(format!(
"unsupported AES key length {}",
key.len()
)));
}
};
rmp_serde::from_slice::<AssetBundleInfo>(decrypted)
.map_err(|err| AssetExecutionError::AssetInfoDecode(err.to_string()))
}
pub fn deobfuscate(data: &[u8]) -> Vec<u8> {
const SIMPLE: [u8; 4] = [0x20, 0x00, 0x00, 0x00];
const XOR_HEADER: [u8; 4] = [0x10, 0x00, 0x00, 0x00];
if data.starts_with(&SIMPLE) {
return data[4..].to_vec();
}
if data.starts_with(&XOR_HEADER) {
let body = &data[4..];
if body.len() < 128 {
return body.to_vec();
}
let mut header = vec![0u8; 128];
let pattern = [0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00];
for idx in 0..128 {
header[idx] = body[idx] ^ pattern[idx % pattern.len()];
}
let mut output = header;
output.extend_from_slice(&body[128..]);
return output;
}
data.to_vec()
}
fn time_arg_jst() -> String {
let tz = FixedOffset::east_opt(9 * 3600).unwrap();
format!(
"?t={}",
chrono::Utc::now().with_timezone(&tz).format("%Y%m%d%H%M%S")
)
}

89
assets-updater/src/core/codec.rs Normal file
View File

@ -0,0 +1,89 @@
use std::fs::File;
use std::path::{Path, PathBuf};
use cridecoder::{extract_acb_from_file, extract_usm_file, HcaDecoder};
use serde::Serialize;
use crate::core::errors::CodecError;
pub const CODEC_BACKEND: &str = "crates.io:cridecoder@0.1.1";
#[derive(Debug, Clone, Serialize)]
pub struct CodecSummary {
pub backend: &'static str,
pub supports_acb: bool,
pub supports_usm: bool,
pub supports_hca_to_wav: bool,
pub supports_usm_metadata: bool,
}
pub fn codec_summary() -> CodecSummary {
CodecSummary {
backend: CODEC_BACKEND,
supports_acb: true,
supports_usm: true,
supports_hca_to_wav: true,
supports_usm_metadata: true,
}
}
pub fn export_acb(input: &Path, output_dir: &Path) -> Result<Option<Vec<String>>, CodecError> {
extract_acb_from_file(input, output_dir).map_err(|err| CodecError::Acb(err.to_string()))
}
pub fn export_usm(input: &Path, output_dir: &Path) -> Result<Vec<PathBuf>, CodecError> {
let outputs = extract_usm_file(input, output_dir, None, false)
.map_err(|err| CodecError::Usm(err.to_string()))?;
normalize_usm_output_names(input, outputs)
}
pub fn read_usm_metadata(input: &Path) -> Result<cridecoder::usm::Metadata, CodecError> {
cridecoder::usm::read_metadata_file(input).map_err(|err| CodecError::Metadata(err.to_string()))
}
pub fn decode_hca_to_wav(input: &Path, output: &Path) -> Result<(), CodecError> {
let input_path = input
.to_str()
.ok_or_else(|| CodecError::NonUtf8Path(input.to_path_buf()))?;
let mut decoder =
HcaDecoder::from_file(input_path).map_err(|err| CodecError::Hca(err.to_string()))?;
let mut file = File::create(output).map_err(|source| CodecError::Io {
path: output.to_path_buf(),
source,
})?;
decoder
.decode_to_wav(&mut file)
.map_err(|err| CodecError::Hca(err.to_string()))
}
fn normalize_usm_output_names(
input: &Path,
outputs: Vec<PathBuf>,
) -> Result<Vec<PathBuf>, CodecError> {
let input_stem = input
.file_stem()
.and_then(|stem| stem.to_str())
.ok_or_else(|| CodecError::NonUtf8Path(input.to_path_buf()))?;
let mut normalized = Vec::with_capacity(outputs.len());
for output in outputs {
let ext = output
.extension()
.and_then(|ext| ext.to_str())
.ok_or_else(|| CodecError::NonUtf8Path(output.clone()))?;
let target = output.with_file_name(format!("{input_stem}.{ext}"));
if output != target {
std::fs::rename(&output, &target).map_err(|source| CodecError::Io {
path: target.clone(),
source,
})?;
normalized.push(target);
} else {
normalized.push(output);
}
}
Ok(normalized)
}

167
assets-updater/src/core/config.rs Normal file
View File

@ -0,0 +1,167 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::core::errors::ConfigError;
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct AppConfig {
pub execution: ExecutionConfig,
pub tools: ToolsConfig,
pub concurrency: ConcurrencyConfig,
pub regions: BTreeMap<String, RegionConfig>,
}
impl AppConfig {
pub fn validate(&self) -> Result<(), ConfigError> {
for region_name in self.regions.keys() {
if region_name.to_lowercase() != *region_name {
return Err(ConfigError::InvalidRegionName(region_name.clone()));
}
}
Ok(())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct ToolsConfig {
pub ffmpeg_path: String,
pub asset_studio_cli_path: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct ExecutionConfig {
pub proxy: Option<String>,
pub timeout_seconds: u64,
pub allow_cancel: bool,
pub retry: RetryConfig,
}
impl Default for ExecutionConfig {
fn default() -> Self {
Self {
proxy: None,
timeout_seconds: 300,
allow_cancel: true,
retry: RetryConfig::default(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct RetryConfig {
pub attempts: usize,
pub initial_backoff_ms: u64,
pub max_backoff_ms: u64,
}
impl Default for RetryConfig {
fn default() -> Self {
Self {
attempts: 4,
initial_backoff_ms: 1_000,
max_backoff_ms: 4_000,
}
}
}
impl Default for ToolsConfig {
fn default() -> Self {
Self {
ffmpeg_path: "ffmpeg".to_string(),
asset_studio_cli_path: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct ConcurrencyConfig {
pub download: usize,
pub upload: usize,
pub acb: usize,
pub usm: usize,
pub hca: usize,
}
impl Default for ConcurrencyConfig {
fn default() -> Self {
Self {
download: 4,
upload: 4,
acb: 8,
usm: 4,
hca: 16,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
pub struct RegionConfig {
pub provider: RegionProviderConfig,
pub crypto: CryptoConfig,
pub runtime: RegionRuntimeConfig,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum RegionProviderConfig {
ColorfulPalette {
asset_info_url_template: String,
asset_bundle_url_template: String,
profile_hash: String,
},
Nuverse {
asset_version_url: String,
asset_info_url_template: String,
asset_bundle_url_template: String,
#[serde(default)]
required_cookies: bool,
#[serde(default)]
cookie_bootstrap_url: Option<String>,
},
}
impl Default for RegionProviderConfig {
fn default() -> Self {
Self::ColorfulPalette {
asset_info_url_template: String::new(),
asset_bundle_url_template: String::new(),
profile_hash: String::new(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
pub struct CryptoConfig {
pub aes_key_hex: Option<String>,
pub aes_iv_hex: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct RegionRuntimeConfig {
pub unity_version: String,
}
impl Default for RegionRuntimeConfig {
fn default() -> Self {
Self {
unity_version: "2022.3.21f1".to_string(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
pub struct RegionPathsConfig {
pub asset_save_dir: Option<String>,
pub downloaded_asset_record_file: Option<String>,
}

144
assets-updater/src/core/errors.rs Normal file
View File

@ -0,0 +1,144 @@
use std::path::PathBuf;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum ConfigError {
#[error("failed to read config file {path}: {source}")]
Read {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("failed to parse config file {path}: {source}")]
Parse {
path: PathBuf,
#[source]
source: yaml_serde::Error,
},
#[error("config_version must be 2, got {0}")]
UnsupportedVersion(u32),
#[error("no v2 config file found; tried: {0}")]
MissingConfigFile(String),
#[error("invalid region key `{0}`; region keys must be lowercase")]
InvalidRegionName(String),
#[error("missing required environment variable `{name}` referenced by `{field}`")]
MissingEnvironmentVariable { field: String, name: String },
}
#[derive(Debug, Error)]
pub enum RegionError {
#[error("region `{0}` not found")]
NotFound(String),
#[error("region `{0}` is disabled")]
Disabled(String),
}
#[derive(Debug, Error)]
pub enum CodecError {
#[error("path {0} is not valid UTF-8 for cridecoder file APIs")]
NonUtf8Path(PathBuf),
#[error("io error at {path}: {source}")]
Io {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("ACB extraction failed: {0}")]
Acb(String),
#[error("USM extraction failed: {0}")]
Usm(String),
#[error("USM metadata read failed: {0}")]
Metadata(String),
#[error("HCA decode failed: {0}")]
Hca(String),
}
#[derive(Debug, Error)]
pub enum ExportPipelineError {
#[error(transparent)]
Codec(#[from] CodecError),
#[error("io error at {path}: {source}")]
Io {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("image codec error at {path}: {source}")]
Image {
path: PathBuf,
#[source]
source: image::ImageError,
},
#[error("failed to spawn command `{program}`: {source}")]
Spawn {
program: String,
#[source]
source: std::io::Error,
},
#[error("command `{program}` failed with status {status}: {stderr}")]
CommandFailed {
program: String,
status: String,
stderr: String,
},
#[error("failed to spawn worker `{worker}`: {source}")]
WorkerSpawn {
worker: String,
#[source]
source: std::io::Error,
},
#[error("worker `{worker}` panicked: {message}")]
WorkerPanic { worker: String, message: String },
}
#[derive(Debug, Error)]
pub enum AssetExecutionError {
#[error(transparent)]
Region(#[from] RegionError),
#[error(transparent)]
ExportPipeline(#[from] ExportPipelineError),
#[error("http request failed: {0}")]
Http(#[from] reqwest::Error),
#[error("failed to initialize HTTP client: {0}")]
HttpClient(String),
#[error("HTTP request to {url} returned status {status}")]
HttpStatus { url: String, status: u16 },
#[error("region `{region}` is missing asset_save_dir")]
MissingAssetSaveDir { region: String },
#[error("nuverse region `{region}` requires asset_version and asset_hash")]
MissingAppVersion { region: String },
#[error("colorful_palette region `{region}` requires asset_version and asset_hash")]
MissingAssetVersionOrHash { region: String },
#[error("colorful_palette region `{region}` is missing profile hash for `{profile}`")]
MissingProfileHash { region: String, profile: String },
#[error("region `{region}` is missing AES key or IV configuration")]
MissingCryptoConfig { region: String },
#[error("invalid AES key hex: {0}")]
InvalidAesKeyHex(String),
#[error("invalid AES IV hex: {0}")]
InvalidAesIvHex(String),
#[error("invalid AES IV length: got {got}, want 16")]
InvalidAesIvLength { got: usize },
#[error("encrypted content cannot be empty")]
EmptyEncryptedContent,
#[error("encrypted content length is not a multiple of AES block size")]
InvalidEncryptedBlockSize,
#[error("failed to decrypt or deserialize asset info: {0}")]
AssetInfoDecode(String),
#[error("failed to create temp directory for {path}: {source}")]
CreateTempDir {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("failed to write temp file {path}: {source}")]
WriteTempFile {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("job execution cancelled")]
Cancelled,
}

774
assets-updater/src/core/export_pipeline.rs Normal file
View File

@ -0,0 +1,774 @@
use std::collections::VecDeque;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use crate::core::codec;
use crate::core::config::{AppConfig, RegionConfig};
use crate::core::errors::ExportPipelineError;
use crate::core::media::{
FrameRate, convert_m2v_to_mp4, convert_usm_to_mp4, convert_wav_to_flac, convert_wav_to_mp3,
};
use crate::core::retry::retry_async;
use common::updater::SyncContext;
use image::codecs::webp::WebPEncoder;
use image::{ExtendedColorType, ImageReader};
use log::debug;
use serde::Serialize;
use tokio::process::Command;
#[derive(Debug, Clone, Copy)]
struct AssetStudioCliCapabilities {
filter_exclude_mode: bool,
filter_blacklist_mode: bool,
sekai_keep_single_container_filename: bool,
}
#[derive(Debug, Clone, Serialize, Default)]
pub struct PostProcessSummary {
pub export_root: PathBuf,
pub generated_files: Vec<PathBuf>,
}
pub fn get_export_group(export_path: &str) -> &'static str {
if export_path.is_empty() {
return "container";
}
let normalized = export_path
.replace('\\', "/")
.trim_start_matches('/')
.to_lowercase();
for prefix in [
"event/center",
"event/thumbnail",
"gacha/icon",
"fix_prefab/mc_new",
"mysekai/character/",
] {
if normalized.starts_with(prefix) {
return "containerFull";
}
}
"container"
}
pub fn get_hex_index(input: &str) -> String {
let hash_val = twox_hash::XxHash3_128::oneshot(input.as_bytes());
format!("{:032x}", hash_val)
}
pub fn empty_dir(base: PathBuf, name: String) -> PathBuf {
let mut dir = base.join(&name);
let mut cnt = 1;
while dir.exists() {
dir = base.join(format!("{}_{}", &name, cnt));
cnt += 1;
}
std::fs::create_dir_all(&dir).unwrap();
dir
}
pub async fn extract_unity_asset_bundle(
app_config: &AppConfig,
sync_context: &SyncContext,
region: &RegionConfig,
asset_bundle_file: &Path,
export_path: &str,
download_path: &str,
) -> Result<(PathBuf, bool), ExportPipelineError> {
let hash = get_hex_index(export_path);
let output_dir = empty_dir(
std::env::temp_dir()
.join("sekai-updater")
.join("extract")
.join(&sync_context.region),
hash,
);
let Some(asset_studio_cli_path) = app_config.tools.asset_studio_cli_path.as_deref() else {
return Ok((asset_bundle_file.parent().unwrap().to_path_buf(), true));
};
let exclude_path_prefix = "assets/sekai/assetbundle/resources/ondemand".to_string();
let actual_export_path = output_dir.join(export_path);
debug!("{}", actual_export_path.to_string_lossy());
let capabilities = detect_assetstudio_cli_capabilities(asset_studio_cli_path);
let args = build_assetstudio_export_args(
asset_bundle_file,
output_dir.as_path(),
export_path,
&exclude_path_prefix,
region,
sync_context,
capabilities,
);
retry_async(
&app_config.execution.retry,
"assetstudio export",
|_| async {
let output = Command::new(asset_studio_cli_path)
.args(&args)
.output()
.await
.map_err(|source| ExportPipelineError::Spawn {
program: asset_studio_cli_path.to_string(),
source,
})?;
if output.status.success() {
Ok(())
} else {
Err(ExportPipelineError::CommandFailed {
program: asset_studio_cli_path.to_string(),
status: output.status.to_string(),
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
})
}
},
is_retryable_command_error,
)
.await?;
let mut count = 0;
let mut is_bundle_name_file = true;
let download_name = match download_path.rsplit_once("/") {
None => download_path,
Some((_, name)) => name,
};
walk(&output_dir, &mut |f| {
count += 1;
let file_name = f.file_name().unwrap().to_string_lossy();
is_bundle_name_file = is_bundle_name_file
&& match file_name.rsplit_once(".") {
Some((name, _)) => name.eq_ignore_ascii_case(download_name),
None => file_name.eq_ignore_ascii_case(download_name),
}
})?;
post_process_exported_files(app_config, sync_context, region, &output_dir).await?;
Ok((output_dir, is_bundle_name_file && count <= 1))
}
pub async fn post_process_exported_files(
app_config: &AppConfig,
sync_context: &SyncContext,
region: &RegionConfig,
export_path: &Path,
) -> Result<(), ExportPipelineError> {
if !export_path.exists() {
return Ok(());
}
handle_usm_files(
export_path,
sync_context,
&app_config.tools.ffmpeg_path,
&app_config.execution.retry,
)
.await?;
handle_acb_files(
export_path,
sync_context,
region,
&app_config.tools.ffmpeg_path,
&app_config.execution.retry,
app_config.concurrency.acb,
app_config.concurrency.hca,
)
.await?;
handle_png_conversion(export_path, sync_context, region).await?;
Ok(())
}
async fn handle_usm_files(
export_path: &Path,
sync_context: &SyncContext,
ffmpeg_path: &str,
retry: &crate::core::config::RetryConfig,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let usm_files = find_files_by_extension(export_path, "usm")?;
if !sync_context.export.usm.export || !sync_context.export.usm.decode || usm_files.is_empty() {
return Ok(Vec::new());
}
let mut out: Vec<PathBuf> = vec![];
for f in usm_files {
out.append(&mut process_usm_file(&f, sync_context, ffmpeg_path, retry).await?);
}
Ok(out)
}
async fn process_usm_file(
usm_file: &Path,
sync_context: &SyncContext,
ffmpeg_path: &str,
retry: &crate::core::config::RetryConfig,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let output_name = usm_file
.file_stem()
.and_then(|stem| stem.to_str())
.ok_or_else(|| ExportPipelineError::Io {
path: usm_file.to_path_buf(),
source: std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid usm file name"),
})?
.to_string();
if sync_context.export.video.convert_to_mp4
&& sync_context.export.video.direct_usm_to_mp4_with_ffmpeg
{
let mp4 = usm_file
.parent()
.unwrap()
.join(format!("{output_name}.mp4"));
convert_usm_to_mp4(usm_file, &mp4, ffmpeg_path, retry).await?;
remove_file_if_exists(usm_file)?;
return Ok(vec![mp4]);
}
let metadata = codec::read_usm_metadata(usm_file).ok();
let frame_rate = metadata
.as_ref()
.and_then(|metadata| metadata.video_frame_rate())
.filter(|(_, denominator)| *denominator > 0)
.map(FrameRate::from_tuple);
let extracted = codec::export_usm(usm_file, usm_file.parent().unwrap())?;
let mut generated = extracted.clone();
if sync_context.export.video.convert_to_mp4 {
for extracted_file in extracted {
if extracted_file
.extension()
.and_then(|ext| ext.to_str())
.map(|ext| ext.eq_ignore_ascii_case("m2v"))
.unwrap_or(false)
{
let mp4 = usm_file
.parent()
.unwrap()
.join(format!("{output_name}.mp4"));
convert_m2v_to_mp4(
&extracted_file,
&mp4,
sync_context.export.video.remove_m2v,
ffmpeg_path,
frame_rate,
retry,
)
.await?;
generated.push(mp4);
if sync_context.export.video.remove_m2v {
generated.retain(|path| path != &extracted_file);
}
}
}
}
remove_file_if_exists(usm_file)?;
Ok(generated)
}
async fn handle_acb_files(
export_path: &Path,
sync_context: &SyncContext,
region: &RegionConfig,
ffmpeg_path: &str,
retry: &crate::core::config::RetryConfig,
acb_concurrency: usize,
hca_concurrency: usize,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let acb_files = find_files_by_extension(export_path, "acb")?;
if !sync_context.export.acb.export || !sync_context.export.acb.decode || acb_files.is_empty() {
return Ok(Vec::new());
}
let export_path = export_path.to_path_buf();
let region = region.clone();
let sync_context = sync_context.clone();
let ffmpeg_path = ffmpeg_path.to_string();
let retry = retry.clone();
run_path_tasks(acb_files, acb_concurrency, move |acb_file| {
process_acb_file(
&acb_file,
&export_path,
&sync_context,
&region,
&ffmpeg_path,
&retry,
hca_concurrency,
)
})
}
fn process_acb_file(
acb_file: &Path,
_output_dir: &Path,
sync_context: &SyncContext,
region: &RegionConfig,
ffmpeg_path: &str,
retry: &crate::core::config::RetryConfig,
hca_concurrency: usize,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let parent_dir = acb_file.parent().ok_or_else(|| ExportPipelineError::Io {
path: acb_file.to_path_buf(),
source: std::io::Error::new(std::io::ErrorKind::NotFound, "missing parent directory"),
})?;
let extract_dir = tempfile::Builder::new()
.prefix("acb-extract-")
.tempdir_in(parent_dir)
.map_err(|source| ExportPipelineError::Io {
path: parent_dir.to_path_buf(),
source,
})?;
let _ = codec::export_acb(acb_file, extract_dir.path())?;
let mut hca_files = find_files_by_extension(extract_dir.path(), "hca")?;
let acb_path_lower = acb_file.to_string_lossy().replace('\\', "/").to_lowercase();
if acb_path_lower.contains("music/long") {
hca_files.retain(|path| {
let lower = path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_default()
.to_lowercase();
!(lower.ends_with("_vr.hca") || lower.ends_with("_screen.hca"))
});
}
if !sync_context.export.hca.decode {
remove_file_if_exists(acb_file)?;
return Ok(Vec::new());
}
let extract_output_dir = extract_dir.path().to_path_buf();
let region = region.clone();
let sync_context = sync_context.clone();
let ffmpeg_path = ffmpeg_path.to_string();
let retry = retry.clone();
let generated = run_path_tasks(hca_files, hca_concurrency, move |hca_file| {
process_hca_file(
&hca_file,
&extract_output_dir,
&sync_context,
&region,
&ffmpeg_path,
&retry,
)
})?;
let final_outputs = move_result_files(acb_file.parent().unwrap(), &generated)?;
remove_file_if_exists(acb_file)?;
Ok(final_outputs)
}
fn process_hca_file(
hca_file: &Path,
output_dir: &Path,
sync_context: &SyncContext,
_: &RegionConfig,
ffmpeg_path: &str,
retry: &crate::core::config::RetryConfig,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let base_name = hca_file
.file_stem()
.and_then(|stem| stem.to_str())
.ok_or_else(|| ExportPipelineError::Io {
path: hca_file.to_path_buf(),
source: std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid hca file name"),
})?;
let wav_file = hca_file.with_extension("wav");
codec::decode_hca_to_wav(hca_file, &wav_file)?;
remove_file_if_exists(hca_file)?;
let mut generated = vec![wav_file.clone()];
if sync_context.export.audio.convert_to_mp3 {
let mp3 = output_dir.join(format!("{base_name}.mp3"));
convert_wav_to_mp3(&wav_file, &mp3, ffmpeg_path, retry)?;
if sync_context.export.audio.remove_wav {
remove_file_if_exists(&wav_file)?;
generated.retain(|path| path != &wav_file);
}
generated.push(mp3);
} else if sync_context.export.audio.convert_to_flac {
let flac = output_dir.join(format!("{base_name}.flac"));
convert_wav_to_flac(&wav_file, &flac, ffmpeg_path, retry)?;
if sync_context.export.audio.remove_wav {
remove_file_if_exists(&wav_file)?;
generated.retain(|path| path != &wav_file);
}
generated.push(flac);
} else if sync_context.export.audio.remove_wav {
remove_file_if_exists(&wav_file)?;
generated.clear();
}
let final_outputs = move_result_files(hca_file.parent().unwrap(), &generated)?;
Ok(final_outputs)
}
async fn handle_png_conversion(
export_path: &Path,
sync_context: &SyncContext,
_: &RegionConfig,
) -> Result<Vec<PathBuf>, ExportPipelineError> {
if !sync_context.export.images.convert_to_webp {
return Ok(Vec::new());
}
let png_files = find_files_by_extension(export_path, "png")?;
let mut generated = Vec::new();
for png_file in png_files {
let webp = png_file.with_extension("webp");
convert_png_to_webp(&png_file, &webp)?;
generated.push(webp.clone());
if sync_context.export.images.remove_png {
remove_file_if_exists(&png_file)?;
}
}
Ok(generated)
}
fn convert_png_to_webp(png_file: &Path, webp_file: &Path) -> Result<(), ExportPipelineError> {
let image = ImageReader::open(png_file)
.map_err(|source| ExportPipelineError::Io {
path: png_file.to_path_buf(),
source,
})?
.decode()
.map_err(|source| ExportPipelineError::Image {
path: png_file.to_path_buf(),
source,
})?;
let rgba = image.to_rgba8();
let (width, height) = rgba.dimensions();
let writer = std::fs::File::create(webp_file).map_err(|source| ExportPipelineError::Io {
path: webp_file.to_path_buf(),
source,
})?;
let writer = std::io::BufWriter::new(writer);
WebPEncoder::new_lossless(writer)
.encode(rgba.as_raw(), width, height, ExtendedColorType::Rgba8)
.map_err(|source| ExportPipelineError::Image {
path: webp_file.to_path_buf(),
source,
})
}
fn is_retryable_command_error(err: &ExportPipelineError) -> bool {
match err {
ExportPipelineError::Spawn { source, .. } => matches!(
source.kind(),
std::io::ErrorKind::Interrupted
| std::io::ErrorKind::TimedOut
| std::io::ErrorKind::WouldBlock
| std::io::ErrorKind::BrokenPipe
| std::io::ErrorKind::ConnectionReset
| std::io::ErrorKind::ConnectionAborted
| std::io::ErrorKind::ConnectionRefused
),
ExportPipelineError::CommandFailed { .. } => true,
_ => false,
}
}
fn build_assetstudio_export_args(
asset_bundle_file: &Path,
output_dir: &Path,
export_path: &str,
exclude_path_prefix: &str,
region: &RegionConfig,
sync_context: &SyncContext,
capabilities: AssetStudioCliCapabilities,
) -> Vec<String> {
let mut args = vec![
asset_bundle_file.to_string_lossy().to_string(),
"-m".to_string(),
"export".to_string(),
"-t".to_string(),
"monoBehaviour,textAsset,tex2d,tex2dArray,audio".to_string(),
"-g".to_string(),
get_export_group(export_path).to_string(),
"-f".to_string(),
"assetName".to_string(),
"-o".to_string(),
output_dir.to_string_lossy().to_string(),
"--strip-path-prefix".to_string(),
exclude_path_prefix.to_string(),
"-r".to_string(),
];
if capabilities.filter_exclude_mode {
args.push("--filter-exclude-mode".to_string());
} else if capabilities.filter_blacklist_mode {
args.push("--filter-blacklist-mode".to_string());
}
args.push("--filter-with-regex".to_string());
if capabilities.sekai_keep_single_container_filename {
args.push("--sekai-keep-single-container-filename".to_string());
}
if !region.runtime.unity_version.is_empty() {
args.push("--unity-version".to_string());
args.push(region.runtime.unity_version.clone());
}
let mut excluded_exts = Vec::new();
if !sync_context.export.usm.export {
excluded_exts.push("usm");
}
if !sync_context.export.acb.export {
excluded_exts.push("acb");
}
if !excluded_exts.is_empty() {
args.push("--filter-by-name".to_string());
args.push(format!(r".*\.({})$", excluded_exts.join("|")));
}
args
}
fn detect_assetstudio_cli_capabilities(asset_studio_cli_path: &str) -> AssetStudioCliCapabilities {
static CACHE: std::sync::OnceLock<
Mutex<std::collections::HashMap<String, AssetStudioCliCapabilities>>,
> = std::sync::OnceLock::new();
let cache = CACHE.get_or_init(|| Mutex::new(std::collections::HashMap::new()));
if let Some(cached) = cache.lock().unwrap().get(asset_studio_cli_path).copied() {
return cached;
}
let fallback = AssetStudioCliCapabilities {
filter_exclude_mode: true,
filter_blacklist_mode: false,
sekai_keep_single_container_filename: true,
};
let detected = match std::process::Command::new(asset_studio_cli_path)
.arg("--help")
.output()
{
Ok(output) => {
let help = format!(
"{}\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
AssetStudioCliCapabilities {
filter_exclude_mode: help.contains("--filter-exclude-mode"),
filter_blacklist_mode: help.contains("--filter-blacklist-mode"),
sekai_keep_single_container_filename: help
.contains("--sekai-keep-single-container-filename"),
}
}
Err(_) => fallback,
};
cache
.lock()
.unwrap()
.insert(asset_studio_cli_path.to_string(), detected);
detected
}
fn move_result_files(
output_dir: &Path,
generated: &[PathBuf],
) -> Result<Vec<PathBuf>, ExportPipelineError> {
let mut final_outputs = Vec::new();
for path in generated {
let file_name = match path.file_name() {
Some(name) => name,
None => continue,
};
let destination = output_dir.join(file_name);
if path != &destination {
std::fs::rename(path, &destination).map_err(|source| ExportPipelineError::Io {
path: destination.clone(),
source,
})?;
final_outputs.push(destination);
} else if destination.exists() {
final_outputs.push(destination);
}
}
Ok(final_outputs)
}
fn run_path_tasks<F>(
paths: Vec<PathBuf>,
concurrency: usize,
task: F,
) -> Result<Vec<PathBuf>, ExportPipelineError>
where
F: Fn(PathBuf) -> Result<Vec<PathBuf>, ExportPipelineError> + Send + Sync + 'static,
{
if paths.is_empty() {
return Ok(Vec::new());
}
let worker_count = concurrency.max(1).min(paths.len());
let queue = Arc::new(Mutex::new(VecDeque::from(paths)));
let results = Arc::new(Mutex::new(Vec::<PathBuf>::new()));
let first_error = Arc::new(Mutex::new(None::<ExportPipelineError>));
let task = Arc::new(task);
let mut handles = Vec::with_capacity(worker_count);
const WORKER_STACK_SIZE: usize = 32 * 1024 * 1024;
for _ in 0..worker_count {
let queue = queue.clone();
let results = results.clone();
let first_error = first_error.clone();
let task = task.clone();
let worker_name = "export-task".to_string();
let handle = std::thread::Builder::new()
.name(worker_name.clone())
.stack_size(WORKER_STACK_SIZE)
.spawn(move || {
loop {
if first_error.lock().unwrap().is_some() {
break;
}
let next_path = queue.lock().unwrap().pop_front();
let Some(path) = next_path else {
break;
};
match task(path) {
Ok(mut generated) => results.lock().unwrap().append(&mut generated),
Err(err) => {
let mut first = first_error.lock().unwrap();
if first.is_none() {
*first = Some(err);
}
break;
}
}
}
})
.map_err(|source| ExportPipelineError::WorkerSpawn {
worker: worker_name,
source,
})?;
handles.push(handle);
}
for handle in handles {
handle
.join()
.map_err(|panic| ExportPipelineError::WorkerPanic {
worker: "export task".to_string(),
message: panic_message(panic),
})?;
}
if let Some(err) = first_error.lock().unwrap().take() {
return Err(err);
}
let mut results = results.lock().unwrap();
Ok(std::mem::take(&mut *results))
}
fn panic_message(panic: Box<dyn std::any::Any + Send>) -> String {
if let Some(message) = panic.downcast_ref::<&str>() {
(*message).to_string()
} else if let Some(message) = panic.downcast_ref::<String>() {
message.clone()
} else {
"unknown worker panic".to_string()
}
}
pub fn find_files(dir: &Path) -> Result<Vec<PathBuf>, ExportPipelineError> {
let mut files = Vec::new();
walk(dir, &mut |path| {
files.push(path.to_path_buf());
})?;
Ok(files)
}
pub fn find_files_by_extension(dir: &Path, ext: &str) -> Result<Vec<PathBuf>, ExportPipelineError> {
let target_ext = ext.to_lowercase();
let mut files = Vec::new();
walk(dir, &mut |path| {
if path
.extension()
.and_then(|value| value.to_str())
.map(|value| value.eq_ignore_ascii_case(&target_ext))
.unwrap_or(false)
{
files.push(path.to_path_buf());
}
})?;
Ok(files)
}
pub fn find_files_by_extensions<P>(
dir: &Path,
ext: &[P],
) -> Result<Vec<PathBuf>, ExportPipelineError>
where
P: AsRef<str>,
{
let target_ext = ext
.iter()
.map(|x| x.as_ref().to_lowercase())
.collect::<Vec<_>>();
let mut files = Vec::new();
walk(dir, &mut |path| {
if path
.extension()
.and_then(|value| value.to_str())
.map(|value| target_ext.contains(&value.to_lowercase()))
.unwrap_or(false)
{
files.push(path.to_path_buf());
}
})?;
Ok(files)
}
fn walk(dir: &Path, f: &mut dyn FnMut(&Path)) -> Result<(), ExportPipelineError> {
for entry in std::fs::read_dir(dir).map_err(|source| ExportPipelineError::Io {
path: dir.to_path_buf(),
source,
})? {
let entry = entry.map_err(|source| ExportPipelineError::Io {
path: dir.to_path_buf(),
source,
})?;
let path = entry.path();
let file_type = entry
.file_type()
.map_err(|source| ExportPipelineError::Io {
path: path.clone(),
source,
})?;
if file_type.is_dir() {
walk(&path, f)?;
} else {
f(&path);
}
}
Ok(())
}
fn remove_file_if_exists(path: &Path) -> Result<(), ExportPipelineError> {
if path.exists() {
std::fs::remove_file(path).map_err(|source| ExportPipelineError::Io {
path: path.to_path_buf(),
source,
})?;
}
Ok(())
}

229
assets-updater/src/core/media.rs Normal file
View File

@ -0,0 +1,229 @@
use std::fmt::{Display, Formatter};
use std::path::Path;
use tokio::process::Command;
use crate::core::config::RetryConfig;
use crate::core::errors::ExportPipelineError;
use crate::core::retry::{retry_async, retry_sync};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct FrameRate {
pub numerator: i32,
pub denominator: i32,
}
impl FrameRate {
pub fn from_tuple((numerator, denominator): (i32, i32)) -> Self {
Self {
numerator,
denominator,
}
}
}
impl Display for FrameRate {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
if self.denominator <= 1 {
write!(f, "{}", self.numerator)
} else {
write!(f, "{}/{}", self.numerator, self.denominator)
}
}
}
pub async fn convert_usm_to_mp4(
usm_file: &Path,
mp4_file: &Path,
ffmpeg_path: &str,
retry: &RetryConfig,
) -> Result<(), ExportPipelineError> {
retry_async(
retry,
"ffmpeg usm->mp4",
|_| async {
run_ffmpeg(
ffmpeg_path,
&[
"-i",
&usm_file.to_string_lossy(),
"-c:v",
"libx264",
"-c:a",
"aac",
"-b:a",
"192k",
"-movflags",
"+faststart",
"-y",
&mp4_file.to_string_lossy(),
],
)
.run_async()
.await
},
is_retryable_command_error,
)
.await
}
pub async fn convert_m2v_to_mp4(
m2v_file: &Path,
mp4_file: &Path,
delete_original: bool,
ffmpeg_path: &str,
frame_rate: Option<FrameRate>,
retry: &RetryConfig,
) -> Result<(), ExportPipelineError> {
let mut args = Vec::new();
if let Some(rate) = frame_rate {
args.push("-r".to_string());
args.push(rate.to_string());
}
args.push("-i".to_string());
args.push(m2v_file.to_string_lossy().to_string());
args.push("-c:v".to_string());
args.push("libx264".to_string());
if let Some(rate) = frame_rate {
args.push("-r".to_string());
args.push(rate.to_string());
}
args.push("-y".to_string());
args.push(mp4_file.to_string_lossy().to_string());
retry_async(
retry,
"ffmpeg m2v->mp4",
|_| async {
let refs: Vec<&str> = args.iter().map(String::as_str).collect();
run_ffmpeg(ffmpeg_path, &refs).run_async().await
},
is_retryable_command_error,
)
.await?;
if delete_original && m2v_file.exists() {
std::fs::remove_file(m2v_file).map_err(|source| ExportPipelineError::Io {
path: m2v_file.to_path_buf(),
source,
})?;
}
Ok(())
}
pub fn convert_wav_to_mp3(
wav_file: &Path,
mp3_file: &Path,
ffmpeg_path: &str,
retry: &RetryConfig,
) -> Result<(), ExportPipelineError> {
retry_sync(
retry,
"ffmpeg wav->mp3",
|_| {
run_ffmpeg_sync(
ffmpeg_path,
&[
"-i",
&wav_file.to_string_lossy(),
"-b:a",
"320k",
"-y",
&mp3_file.to_string_lossy(),
],
)
},
is_retryable_command_error,
)
}
pub fn convert_wav_to_flac(
wav_file: &Path,
flac_file: &Path,
ffmpeg_path: &str,
retry: &RetryConfig,
) -> Result<(), ExportPipelineError> {
retry_sync(
retry,
"ffmpeg wav->flac",
|_| {
run_ffmpeg_sync(
ffmpeg_path,
&[
"-i",
&wav_file.to_string_lossy(),
"-compression_level",
"12",
"-y",
&flac_file.to_string_lossy(),
],
)
},
is_retryable_command_error,
)
}
fn run_ffmpeg<'a>(ffmpeg_path: &'a str, args: &'a [&'a str]) -> FfmpegCommand<'a> {
FfmpegCommand { ffmpeg_path, args }
}
fn run_ffmpeg_sync(ffmpeg_path: &str, args: &[&str]) -> Result<(), ExportPipelineError> {
let output = std::process::Command::new(ffmpeg_path)
.args(args)
.output()
.map_err(|source| ExportPipelineError::Spawn {
program: ffmpeg_path.to_string(),
source,
})?;
map_command_output(ffmpeg_path, output)
}
struct FfmpegCommand<'a> {
ffmpeg_path: &'a str,
args: &'a [&'a str],
}
impl<'a> FfmpegCommand<'a> {
async fn run_async(self) -> Result<(), ExportPipelineError> {
let output = Command::new(self.ffmpeg_path)
.args(self.args)
.output()
.await
.map_err(|source| ExportPipelineError::Spawn {
program: self.ffmpeg_path.to_string(),
source,
})?;
map_command_output(self.ffmpeg_path, output)
}
}
fn is_retryable_command_error(err: &ExportPipelineError) -> bool {
match err {
ExportPipelineError::Spawn { source, .. } => matches!(
source.kind(),
std::io::ErrorKind::Interrupted
| std::io::ErrorKind::TimedOut
| std::io::ErrorKind::WouldBlock
| std::io::ErrorKind::BrokenPipe
| std::io::ErrorKind::ConnectionReset
| std::io::ErrorKind::ConnectionAborted
| std::io::ErrorKind::ConnectionRefused
),
ExportPipelineError::CommandFailed { .. } => true,
_ => false,
}
}
fn map_command_output(
program: &str,
output: std::process::Output,
) -> Result<(), ExportPipelineError> {
if output.status.success() {
Ok(())
} else {
Err(ExportPipelineError::CommandFailed {
program: program.to_string(),
status: output.status.to_string(),
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
})
}
}

7
assets-updater/src/core/mod.rs Normal file
View File

@ -0,0 +1,7 @@
pub mod asset_execution;
pub mod config;
pub mod export_pipeline;
pub mod retry;
pub mod media;
pub mod codec;
pub mod errors;

83
assets-updater/src/core/retry.rs Normal file
View File

@ -0,0 +1,83 @@
use std::fmt::Display;
use std::future::Future;
use std::time::Duration;
use log::warn;
use tokio::time::sleep;
use crate::core::config::RetryConfig;
pub async fn retry_async<T, E, Op, Fut, ShouldRetry>(
config: &RetryConfig,
operation: &str,
mut op: Op,
should_retry: ShouldRetry,
) -> Result<T, E>
where
E: Display,
Op: FnMut(usize) -> Fut,
Fut: Future<Output = Result<T, E>>,
ShouldRetry: Fn(&E) -> bool,
{
let attempts = config.attempts.max(1);
for attempt in 1..=attempts {
match op(attempt).await {
Ok(value) => return Ok(value),
Err(err) if attempt < attempts && should_retry(&err) => {
let delay = backoff_delay(config, attempt);
warn!(
"operation '{}' failed after {} attempt(s) with error: {}, retrying in {} ms",
operation,
attempt,
err,
delay.as_millis()
);
sleep(delay).await;
}
Err(err) => return Err(err),
}
}
unreachable!("retry_async must return from within the attempt loop")
}
pub fn retry_sync<T, E, Op, ShouldRetry>(
config: &RetryConfig,
operation: &str,
mut op: Op,
should_retry: ShouldRetry,
) -> Result<T, E>
where
E: Display,
Op: FnMut(usize) -> Result<T, E>,
ShouldRetry: Fn(&E) -> bool,
{
let attempts = config.attempts.max(1);
for attempt in 1..=attempts {
match op(attempt) {
Ok(value) => return Ok(value),
Err(err) if attempt < attempts && should_retry(&err) => {
let delay = backoff_delay(config, attempt);
warn!(
"operation '{}' failed after {} attempt(s) with error: {}, retrying in {} ms",
operation,
attempt,
err,
delay.as_millis()
);
std::thread::sleep(delay);
}
Err(err) => return Err(err),
}
}
unreachable!("retry_sync must return from within the attempt loop")
}
fn backoff_delay(config: &RetryConfig, attempt: usize) -> Duration {
let base = config.initial_backoff_ms.max(1);
let max = config.max_backoff_ms.max(base);
let multiplier = 1u64
.checked_shl(attempt.saturating_sub(1) as u32)
.unwrap_or(u64::MAX);
Duration::from_millis(base.saturating_mul(multiplier).min(max))
}

1
assets-updater/src/lib.rs Normal file
View File

@ -0,0 +1 @@
pub mod core;

28
cargo.toml Normal file
View File

@ -0,0 +1,28 @@
[workspace]
resolver = "3"
members = ["entry", "common", "assets-updater", "test-main"]
#[profile.dev]
#codegen-units = 4
[workspace.dependencies]
anyhow = "1.0.102"
log = "0.4.29"
serde = "1.0.228"
serde_json = "1.0.150"
tokio = "1.51.0"
aes = "0.8.4"
rmp-serde = "1.3.1"
hex = "0.4.3"
cbc = "0.1.2"
tempfile = "3.27.0"
chrono = "0.4.44"
reqwest = "0.13.2"
regex = "1.12.3"
yaml_serde = "0.10.4"
image = "0.25.10"
cridecoder = "0.1.1"
thiserror = "2.0.18"
cipher = "0.4.4"
twox-hash = "2.1.2"

7
common/Cargo.toml Normal file
View File

@ -0,0 +1,7 @@
[package]
name = "common"
version = "0.1.0"
edition = "2024"
[dependencies]
serde = { workspace = true, features = ["derive"] }

1
common/src/lib.rs Normal file
View File

@ -0,0 +1 @@
pub mod updater;

161
common/src/updater.rs Normal file
View File

@ -0,0 +1,161 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyncContext {
pub region: String,
#[serde(default)]
pub export: RegionExportConfig,
#[serde(default)]
pub asset_version: Option<String>,
#[serde(default)]
pub asset_hash: Option<String>,
#[serde(default)]
pub app_version: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
pub struct RegionFiltersConfig {
#[serde(default)]
pub start_app: Vec<String>,
#[serde(default)]
pub on_demand: Vec<String>,
#[serde(default)]
pub skip: Vec<String>,
#[serde(default)]
pub file_ext: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
pub struct RegionExportConfig {
#[serde(default)]
pub usm: UsmExportConfig,
#[serde(default)]
pub acb: AcbExportConfig,
#[serde(default)]
pub hca: HcaExportConfig,
#[serde(default)]
pub images: ImageExportConfig,
#[serde(default)]
pub video: VideoExportConfig,
#[serde(default)]
pub audio: AudioExportConfig,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct UsmExportConfig {
pub export: bool,
pub decode: bool,
}
impl Default for UsmExportConfig {
fn default() -> Self {
Self {
export: true,
decode: true,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct AcbExportConfig {
pub export: bool,
pub decode: bool,
}
impl Default for AcbExportConfig {
fn default() -> Self {
Self {
export: true,
decode: true,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct HcaExportConfig {
pub decode: bool,
}
impl Default for HcaExportConfig {
fn default() -> Self {
Self { decode: true }
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
pub struct ImageExportConfig {
pub convert_to_webp: bool,
pub remove_png: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct VideoExportConfig {
pub convert_to_mp4: bool,
pub direct_usm_to_mp4_with_ffmpeg: bool,
pub remove_m2v: bool,
}
impl Default for VideoExportConfig {
fn default() -> Self {
Self {
convert_to_mp4: true,
direct_usm_to_mp4_with_ffmpeg: false,
remove_m2v: true,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct AudioExportConfig {
pub convert_to_mp3: bool,
pub convert_to_flac: bool,
pub remove_wav: bool,
}
impl Default for AudioExportConfig {
fn default() -> Self {
Self {
convert_to_mp3: true,
convert_to_flac: false,
remove_wav: true,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DownloadTask {
pub download_path: String,
pub bundle_path: String,
pub bundle_hash: String,
pub category: AssetCategory,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub enum AssetCategory {
StartApp,
OnDemand,
Other(String),
}
impl<'de> Deserialize<'de> for AssetCategory {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
// Treat nil/null as Other("") — matches Go's zero-value coercion.
let raw = Option::<String>::deserialize(deserializer)?.unwrap_or_default();
Ok(match raw.as_str() {
"StartApp" | "startApp" => Self::StartApp,
"OnDemand" | "onDemand" => Self::OnDemand,
other => Self::Other(other.to_string()),
})
}
}

22
config.py
View File

@ -1,22 +0,0 @@
from nonebot import get_driver, get_plugin_config
from pydantic import BaseModel
_nickname: str = ""
_proxy: str | None = None
try:
_proxy = next(iter(get_driver().config.proxy), None)
_nickname: str = next(iter(get_driver().config.nickname), "")
except:
pass
class Config(BaseModel):
assets_studio_path: str
github_token: str | None = None
@property
def proxy(self) -> str | None:
return _proxy
@property
def nickname(self) -> str:
return _nickname
pconfig: Config = get_plugin_config(Config)

20
entry/Cargo.toml Normal file
View File

@ -0,0 +1,20 @@
[package]
name = "entry"
version = "0.1.0"
edition = "2024"
[lib]
name = "sekai_sync_lib"
crate-type = ["rlib", "cdylib"]
[dependencies]
common = { path = "../common" }
assets-updater = { path = "../assets-updater" }
tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread"] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
reqwest = { workspace = true, features = ["json"] }
log = { workspace = true }
anyhow = { workspace = true }
simplelog = "0.12.2"

123
entry/src/data.rs Normal file
View File

@ -0,0 +1,123 @@
use crate::CONFIG;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
#[derive(Debug)]
pub struct Config {
pub cwd: String,
pub name: String,
pub as_path: String, // AssetStudio
}
impl Config {
pub fn cache_dir(&self) -> PathBuf {
PathBuf::from(&self.cwd).join("cache").join(&self.name)
}
pub fn data_dir(&self) -> PathBuf {
PathBuf::from(&self.cwd).join("data").join(&self.name)
}
}
#[derive(Debug, Default)]
pub struct OptConfig {
pub proxy: Option<String>,
pub ghp: Option<String>,
}
impl OptConfig {
pub const fn new() -> Self {
Self {
proxy: None,
ghp: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct State {
pub git_hash: Option<String>,
pub master_hash: Option<String>,
pub event_id: Option<u64>,
}
impl State {
fn load(x: &str) -> Self {
serde_json::from_str(x).unwrap()
}
pub const fn new() -> Self {
Self {
git_hash: None,
master_hash: None,
event_id: None,
}
}
pub fn init(&mut self) -> anyhow::Result<()> {
let path = CONFIG.get().unwrap().data_dir().join("state.json");
if path.exists() {
let i = Self::load(&std::fs::read_to_string(path)?);
self.git_hash = i.git_hash;
self.master_hash = i.master_hash;
self.event_id = i.event_id;
}
Ok(())
}
pub fn save(&self) {
let d = CONFIG.get().unwrap().data_dir();
std::fs::create_dir_all(&d).unwrap();
let json = serde_json::to_string(self).unwrap();
std::fs::write(d.join("state.json"), json).unwrap();
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EventStory {
pub id: u64,
#[serde(rename = "eventId")]
pub event_id: u64,
// outline: String,
#[serde(rename = "assetbundleName")]
pub assetbundle_name: String,
}
// 用这个找是因为这个文件小一点（（（
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CardEpisode {
pub id: u64,
pub seq: u64,
#[serde(rename = "cardId")]
pub card_id: u64,
#[serde(rename = "assetbundleName")]
pub assetbundle_name: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EventCard {
pub id: u64,
#[serde(rename = "cardId")]
pub card_id: u64,
#[serde(rename = "eventId")]
pub event_id: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CurrentVersion {
#[serde(rename = "appHash")]
pub app_hash: String,
#[serde(rename = "systemProfile")]
pub system_profile: String,
#[serde(rename = "appVersion")]
pub app_version: String,
#[serde(rename = "assetVersion")]
pub asset_version: String,
#[serde(rename = "dataVersion")]
pub data_version: String,
#[serde(rename = "assetHash")]
pub asset_hash: String,
}
#[derive(Debug, Clone, Serialize)]
pub struct FetchResult{
pub event_id: u64,
pub card_paths: Vec<PathBuf>,
}

60
entry/src/download.rs Normal file
View File

@ -0,0 +1,60 @@
use crate::{CONFIG, OPT_CONFIG};
use crate::github::get_master_data;
use assets_updater::core::asset_execution::AssetExecutionContext;
use assets_updater::core::config::{AppConfig, CryptoConfig, ExecutionConfig, RegionConfig, RegionProviderConfig, RegionRuntimeConfig, ToolsConfig};
use assets_updater::core::errors::AssetExecutionError;
use common::updater::SyncContext;
use std::path::PathBuf;
pub struct Downloader {
exec: AssetExecutionContext,
app_config: AppConfig,
}
impl Downloader {
pub async fn new() -> anyhow::Result<Downloader> {
let master_data = get_master_data().await?;
let mut execution = ExecutionConfig::default();
// if let Ok(opt_conf) = &OPT_CONFIG.read()
// && let Some(proxy) = &opt_conf.proxy
// {
// execution.proxy = Some(proxy.to_string());
// }
let app_config = AppConfig {
execution,
tools: ToolsConfig{
asset_studio_cli_path : Some(CONFIG.get().unwrap().as_path.clone()),
..Default::default()
},
..Default::default()
};
let mut exec = AssetExecutionContext::new(&app_config, &SyncContext{
region: "jp".to_string(),
export: Default::default(),
asset_version: Some(master_data.asset_version),
asset_hash: Some(master_data.asset_hash),
app_version: Some(master_data.app_version),
}, &RegionConfig {
provider: RegionProviderConfig::ColorfulPalette {
asset_info_url_template: "https://{env}-{hash}-assetbundle-info.sekai.colorfulpalette.org/api/version/{asset_version}/{asset_hash}/os/ios".to_owned(),
asset_bundle_url_template: "https://{env}-{hash}-assetbundle.sekai.colorfulpalette.org/{asset_version}/{asset_hash}/ios/{bundle_path}".to_owned(),
profile_hash: "cf2d2388".to_owned(),
},
crypto: CryptoConfig{
aes_iv_hex: Some("6732666343305a637a4e394d544a3631".to_owned()),
aes_key_hex: Some("6d737833495630693958453575595a31".to_owned()),
},
runtime: RegionRuntimeConfig{
unity_version: "2022.3.21f1".to_owned()
},
})?;
exec.fetch_runtime_cookies().await?;
Ok(Downloader { exec, app_config })
}
pub async fn pull(
&self,
bundle_path: &str,
) -> Result<(PathBuf, /*single file*/ bool), AssetExecutionError> {
self.exec.download(bundle_path, &self.app_config).await
}
}

84
entry/src/github.rs Normal file
View File

@ -0,0 +1,84 @@
use crate::data::CurrentVersion;
use crate::{OPT_CONFIG, STORE, UA};
use anyhow::anyhow;
use reqwest::Client;
use serde::Deserialize;
const COMMIT_URL: &str = "https://api.github.com/repos/Team-Haruki/haruki-sekai-master/commits?path=master/events.json&per_page=1";
#[derive(Debug, Deserialize)]
struct CommitResponse {
sha: String,
}
fn http_client() -> anyhow::Result<Client> {
let mut builder = Client::builder();
if let Ok(opt_conf) = &OPT_CONFIG.read() {
if let Some(proxy) = &opt_conf.proxy {
builder = builder.proxy(reqwest::Proxy::all(proxy)?);
}
if let Some(ghp) = &opt_conf.ghp {
builder = builder.default_headers({
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
reqwest::header::AUTHORIZATION,
format!("Bearer {ghp}").parse()?,
);
headers.insert(reqwest::header::USER_AGENT, UA.parse()?);
headers
});
}
}
Ok(builder.build()?)
}
pub async fn fetch_last_hash() -> anyhow::Result<String> {
let resp = http_client()?.get(COMMIT_URL).send().await?;
let text = resp.text().await?;
let data: Vec<CommitResponse> = serde_json::from_str(&text)?;
Ok(data.first().ok_or(anyhow!(""))?.sha.clone())
}
pub fn raw_url(path: &str) -> String {
"https://raw.githubusercontent.com/Team-Haruki/haruki-sekai-master/refs/heads/main/".to_owned()
+ path
}
pub async fn sync_data() -> anyhow::Result<()> {
let client = http_client()?;
let events_cards = &client
.get(raw_url("master/eventCards.json"))
.send()
.await?
.text()
.await?;
let event_stories = &client
.get(raw_url("master/eventStories.json"))
.send()
.await?
.text()
.await?;
let card_episodes = &client
.get(raw_url("master/cardEpisodes.json"))
.send()
.await?
.text()
.await?;
let mut store = STORE.write().map_err(|e| anyhow!("{}", e))?;
store.set_event_cards(serde_json::from_str(events_cards)?);
store.set_event_stories(serde_json::from_str(event_stories)?);
store.set_card_episodes(serde_json::from_str(card_episodes)?);
Ok(())
}
pub async fn get_master_data() -> anyhow::Result<CurrentVersion> {
let client = http_client()?;
let text = client
.get(raw_url("versions/current_version.json"))
.send()
.await?
.text()
.await?;
Ok(serde_json::from_str(&text)?)
}

125
entry/src/lib.rs Normal file
View File

@ -0,0 +1,125 @@
mod data;
mod download;
mod github;
mod main_loop;
mod store;
use crate::data::{Config, FetchResult, OptConfig, State};
use crate::main_loop::main;
use crate::store::Store;
use log::{LevelFilter, debug};
use simplelog::{ColorChoice, TermLogger, TerminalMode};
use std::ffi::{CStr, CString, c_char};
use std::sync::{OnceLock, RwLock};
use std::thread;
use std::time::Duration;
use tokio::runtime::Runtime;
pub static CONFIG: OnceLock<Config> = OnceLock::new();
pub static OPT_CONFIG: RwLock<OptConfig> = RwLock::new(OptConfig::new());
pub static PROVIDE: RwLock<Option<FetchResult>> = RwLock::new(None);
pub static STORE: RwLock<Store> = RwLock::new(Store::new());
pub static STATE: RwLock<State> = RwLock::new(State::new());
pub const UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36";
unsafe fn parse_str(x: *const c_char) -> String {
if x.is_null() {
return "".to_owned();
}
let c_str = unsafe { CStr::from_ptr(x) };
c_str.to_string_lossy().into_owned()
}
macro_rules! def_opt_str_setter {
($name: ident) => {
/// # Safety
#[unsafe(no_mangle)]
pub unsafe extern "C" fn $name(x: *const c_char) {
let x = unsafe { parse_str(x) };
if x.is_empty() {
OPT_CONFIG.write().unwrap().$name = None;
} else {
OPT_CONFIG.write().unwrap().$name = Some(x);
}
}
};
}
def_opt_str_setter!(proxy);
def_opt_str_setter!(ghp);
/// # Safety
#[unsafe(no_mangle)]
pub unsafe extern "C" fn boot(
cwd: *const c_char,
name: *const c_char,
as_path: *const c_char,
) -> i32 {
unsafe {
CONFIG
.set(Config {
cwd: parse_str(cwd),
name: parse_str(name),
as_path: parse_str(as_path),
})
.unwrap();
}
STATE
.write()
.unwrap()
.init()
.expect("Failed to initialize state");
*STORE.write().unwrap() = Store::load();
// TermLogger::init(
// LevelFilter::Debug,
// simplelog::Config::default(),
// TerminalMode::Mixed,
// ColorChoice::Auto,
// )
// .unwrap();
thread::spawn(|| {
let rt = Runtime::new().expect("Failed to create Tokio runtime");
rt.block_on(keep_loop());
});
0
}
/// # Safety
#[unsafe(no_mangle)]
pub unsafe extern "C" fn fetch_data() -> *mut c_char {
let mut provide = PROVIDE.write().unwrap();
if provide.is_none() {
return std::ptr::null_mut();
}
let result = provide.take().unwrap();
*provide = None;
let json_string = serde_json::to_string(&result).unwrap();
let c_str = CString::new(json_string).unwrap();
c_str.into_raw()
}
/// # Safety
#[unsafe(no_mangle)]
pub unsafe extern "C" fn free_string(s: *mut c_char) {
if s.is_null() {
return;
}
unsafe {
let _ = CString::from_raw(s);
}
}
async fn keep_loop() {
loop {
debug!("loop_start");
if let Err(err) = main().await {
println!("Error: {}", err);
}
tokio::time::sleep(Duration::from_secs(60)).await;
}
}

91
entry/src/main_loop.rs Normal file
View File

@ -0,0 +1,91 @@
use crate::data::{FetchResult, State};
use crate::download::Downloader;
use crate::github::{fetch_last_hash, sync_data};
use crate::{CONFIG, PROVIDE, STATE, STORE};
use anyhow::anyhow;
use assets_updater::core::export_pipeline::find_files;
use std::fs;
use log::debug;
pub async fn main() -> anyhow::Result<()> {
let current_hash = fetch_last_hash().await?;
let old_state = {
let mut l = STATE.read();
if l.is_err() {
STATE.clear_poison();
drop(l);
let mut s = STATE.write().map_err(|e| anyhow!("{}", e))?;
*s = State::new();
let _ = s.init();
drop(s);
l = STATE.read();
}
if let Ok(state) = l {
state.clone()
} else {
State::new()
}
};
if let Some(old_hash) = &old_state.git_hash
&& old_hash == &current_hash
{
return Ok(());
}
// hash updated
sync_data().await?;
let store = STORE.read().map_err(|e| anyhow!("{}", e))?;
let event = store.find_last_event();
if event
.as_ref()
.zip(old_state.event_id.as_ref())
.is_none_or(|(e, id)| &e.event_id == id)
{
let mut state = STATE.write().map_err(|e| anyhow!("{}", e))?;
state.git_hash = Some(current_hash);
if let Some(event) = event {
state.event_id = Some(event.event_id);
}
state.save();
return Ok(());
}
let event = event.unwrap();
// event updated
let cards = store.find_cards_by_event(event.event_id);
drop(store);
let downloader = Downloader::new().await?;
let mut images = Vec::new();
let cache_dir = CONFIG.get().unwrap().cache_dir();
for card in cards {
debug!("Pulling card {} from bundle {}", card.card_id, card.assetbundle_name);
let dir = downloader
.pull(&format!("character/member/{}", card.assetbundle_name))
.await?;
let files = find_files(&dir.0)?;
for file in files {
if file.is_file() && file.extension().map(|ext| ext == "png").unwrap_or(false) {
let new_filename = format!(
"{}_{}",
card.assetbundle_name,
file.file_name().unwrap().to_str().unwrap()
);
let new_file = cache_dir.join(new_filename);
fs::copy(&file, &new_file)?;
fs::remove_file(&file)?;
images.push(new_file);
}
}
}
*PROVIDE.write().map_err(|e| anyhow!("{}", e))? = Some(FetchResult {
event_id: event.event_id,
card_paths: images,
});
let mut state = STATE.write().map_err(|e| anyhow!("{}", e))?;
state.git_hash = Some(current_hash);
state.event_id = Some(event.event_id);
state.save();
Ok(())
}

97
entry/src/store.rs Normal file
View File

@ -0,0 +1,97 @@
use crate::CONFIG;
use crate::data::{CardEpisode, EventCard, EventStory};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::fs;
use std::path::PathBuf;
#[derive(Debug, Default)]
pub struct Store {
pub event_stories: Vec<EventStory>,
pub card_episodes: Vec<CardEpisode>,
pub event_cards: Vec<EventCard>,
}
impl Store {
pub const fn new() -> Self {
Self {
event_cards: Vec::new(),
card_episodes: Vec::new(),
event_stories: Vec::new(),
}
}
pub fn load() -> Self {
let cache_dir = CONFIG.get().unwrap().cache_dir();
// 确保缓存目录存在
if !cache_dir.exists() {
let _ = fs::create_dir_all(&cache_dir);
}
Self {
event_stories: Self::load_file(&cache_dir.join("event_stories.json")),
card_episodes: Self::load_file(&cache_dir.join("card_episodes.json")),
event_cards: Self::load_file(&cache_dir.join("event_cards.json")),
}
}
fn load_file<T: DeserializeOwned>(path: &PathBuf) -> Vec<T> {
fs::read_to_string(path)
.ok()
.and_then(|content| serde_json::from_str(&content).ok())
.unwrap_or_default()
}
fn save_file<T: Serialize>(path: &PathBuf, data: &Vec<T>) {
if let Ok(content) = serde_json::to_string_pretty(data) {
let _ = fs::write(path, content);
}
}
pub fn set_event_stories(&mut self, data: Vec<EventStory>) {
self.event_stories = data;
let path = CONFIG.get().unwrap().cache_dir().join("event_stories.json");
Self::save_file(&path, &self.event_stories);
}
pub fn set_card_episodes(&mut self, data: Vec<CardEpisode>) {
self.card_episodes = data;
let path = CONFIG.get().unwrap().cache_dir().join("card_episodes.json");
Self::save_file(&path, &self.card_episodes);
}
pub fn set_event_cards(&mut self, data: Vec<EventCard>) {
self.event_cards = data;
let path = CONFIG.get().unwrap().cache_dir().join("event_cards.json");
Self::save_file(&path, &self.event_cards);
}
pub fn save_all(&self) {
let cache_dir = CONFIG.get().unwrap().cache_dir();
Self::save_file(&cache_dir.join("event_stories.json"), &self.event_stories);
Self::save_file(&cache_dir.join("card_episodes.json"), &self.card_episodes);
Self::save_file(&cache_dir.join("event_cards.json"), &self.event_cards);
}
pub fn find_last_event(&self) -> Option<EventStory> {
self.event_stories
.iter()
.max_by_key(|s| s.event_id)
.cloned()
}
pub fn find_cards_by_event(&self, event_id: u64) -> Vec<CardEpisode> {
self.event_cards
.iter()
.filter(|e| e.event_id == event_id)
.filter_map(|c| {
self.card_episodes
.iter()
.rev()
.find(|e| e.card_id == c.card_id)
})
.cloned()
.collect()
}
}

102
lib.py
View File

@ -1,102 +0,0 @@
import ctypes
import json
import os
import sys
from typing import Optional, Dict, Any, List
def get_lib_filename(base_name):
if sys.platform == "win32":
return f"{base_name}.dll"
elif sys.platform == "darwin":
return f"lib{base_name}.dylib"
else:
return f"lib{base_name}.so"
class FetchData:
event_id: int
card_paths: List[str]
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> 'FetchData':
instance = cls()
instance.event_id = data.get("event_id", 0)
instance.card_paths = data.get("card_paths", [])
return instance
class SekaiSyncLib:
def __init__(self):
self.lib = ctypes.CDLL(os.path.join(os.path.dirname(__file__), get_lib_filename("sekai_sync_lib")))
self._setup_signatures()
def _setup_signatures(self):
# proxy(x: *const c_char)
self.lib.proxy.argtypes = [ctypes.c_char_p]
self.lib.proxy.restype = None
# ghp(x: *const c_char)
self.lib.ghp.argtypes = [ctypes.c_char_p]
self.lib.ghp.restype = None
# boot(cwd: *const c_char, name: *const c_char, as_path: *const c_char) -> i32
self.lib.boot.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p]
self.lib.boot.restype = ctypes.c_int32
# fetch_data() -> *mut c_char
# ⚠️ 这里必须用 c_void_p，不能用 c_char_p，否则会丢失指针地址导致内存泄漏
self.lib.fetch_data.argtypes = []
self.lib.fetch_data.restype = ctypes.c_void_p
# free_string(s: *mut c_char)
self.lib.free_string.argtypes = [ctypes.c_void_p]
self.lib.free_string.restype = None
def _str_to_bytes(self, s: Optional[str]) -> bytes:
"""辅助函数：将 Python 字符串安全转换为 C 字符串字节"""
if s is None:
# 根据 Rust 端解析逻辑，传入空字符串将被视为 None
return b""
return s.encode('utf-8')
def set_proxy(self, proxy_url: Optional[str]):
"""设置代理，传 None 或空字符串可清除"""
self.lib.proxy(self._str_to_bytes(proxy_url))
def set_ghp(self, token: Optional[str]):
"""设置 GitHub Token，传 None 或空字符串可清除"""
self.lib.ghp(self._str_to_bytes(token))
def boot(self, cwd: str, name: str, as_path: str) -> int:
"""
启动主循环与初始化状态
"""
return self.lib.boot(
self._str_to_bytes(cwd),
self._str_to_bytes(name),
self._str_to_bytes(as_path)
)
def fetch_data(self) -> Optional[FetchData]:
# 1. 获取裸指针
ptr = self.lib.fetch_data()
# 指针为空 (std::ptr::null_mut)
if not ptr:
return None
try:
# 2. 将 void_p 转换回 c_char_p 并读取里面的 bytes
json_bytes = ctypes.cast(ptr, ctypes.c_char_p).value
if not json_bytes:
return None
# 3. 解码并反序列化 JSON
json_str = json_bytes.decode('utf-8')
return FetchData.from_dict(json.loads(json_str))
finally:
# 4. 无论是否发生 JSON 解析异常，绝对保证释放 Rust 内存！
self.lib.free_string(ptr)
rlib = SekaiSyncLib()

53
sync.py
View File

@ -1,53 +0,0 @@
import json
from pathlib import Path
from nonebot import get_bot
from nonebot.adapters.onebot.v11 import Message, MessageSegment
from nonebot_plugin_apscheduler import scheduler
from .config import pconfig
from .lib import rlib
group_file = Path("config") / "nonebot_sekai_update_notify" / "group.json"
group_file.parent.mkdir(parents=True, exist_ok=True)
if not group_file.exists():
group_file.write_text("[]")
enabled_groups = json.loads(group_file.read_text())
def add_group(group_id: int):
if group_id not in enabled_groups:
enabled_groups.append(group_id)
group_file.write_text(json.dumps(enabled_groups))
def remove_group(group_id: int):
if group_id in enabled_groups:
enabled_groups.remove(group_id)
group_file.write_text(json.dumps(enabled_groups))
async def sync():
data = rlib.fetch_data()
if data is None:
return
message = Message()
inner = [MessageSegment.text(f"活动 {data.event_id} 的卡牌更新了！\n")]
for item in data.card_paths:
p = Path(item)
n = MessageSegment.image(file=item)
n.data["summary"] = f"[{p.name}]"
inner.append(n)
for node in inner:
message.append(MessageSegment.node_custom(
user_id=int(get_bot().self_id),
nickname=pconfig.nickname,
content=Message([node])
))
for group in enabled_groups:
await get_bot().send_group_msg(message=message, group_id=group)

9
test-main/Cargo.toml Normal file
View File

@ -0,0 +1,9 @@
[package]
name = "test-main"
version = "0.1.0"
edition = "2024"
[dependencies]
entry = { path = "../entry" }
simplelog = "0.12.2"

30
test-main/src/main.rs Normal file
View File

@ -0,0 +1,30 @@
use simplelog::{ColorChoice, Config, LevelFilter, TermLogger, TerminalMode};
use sekai_sync_lib::{boot, ghp, proxy};
fn main() {
let _proxy = "http://127.0.0.1:7890";
let _cwd = "D:\\Workspace\\sekai-sync-lib\\target";
let _name = "sekai-sync-lib";
let _as_path = "D:\\Workspace\\AssetStudio\\AssetStudioCLI\\bin\\Release\\net9.0\\AssetStudioModCLI.exe";
let _ghp = "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";
//
// TermLogger::init(
// LevelFilter::Debug,
// Config::default(),
// TerminalMode::Mixed,
// ColorChoice::Auto,
// ).unwrap();
macro_rules! cstr {
($v:expr) => {
std::ffi::CString::new($v).unwrap().as_ptr()
};
}
unsafe {
proxy(cstr!(_proxy));
ghp(cstr!(_ghp));
boot(cstr!(_cwd), cstr!(_name), cstr!(_as_path));
};
std::thread::sleep(std::time::Duration::from_hours(10));
}