好欸!出!

This commit is contained in:
shenjack 2024-07-24 00:19:17 +08:00
parent 0d432adc29
commit 199d09a292
Signed by: shenjack
GPG Key ID: 7B1134A979775551
6 changed files with 182 additions and 90 deletions

111
Cargo.lock generated
View File

@ -137,9 +137,9 @@ dependencies = [
[[package]]
name = "arrayref"
version = "0.3.7"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545"
checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a"
[[package]]
name = "arrayvec"
@ -166,7 +166,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -177,7 +177,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -314,7 +314,7 @@ dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
"syn_derive",
]
@ -391,9 +391,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.9"
version = "4.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462"
checksum = "8f6b81fb3c84f5563d509c59b5a48d935f689e993afa90fe39047f05adef9142"
dependencies = [
"clap_builder",
"clap_derive",
@ -401,9 +401,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.9"
version = "4.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942"
checksum = "5ca6706fd5224857d9ac5eb9355f6683563cc0541c7cd9d014043b57cbec78ac"
dependencies = [
"anstream",
"anstyle",
@ -420,7 +420,7 @@ dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -743,7 +743,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -980,7 +980,7 @@ dependencies = [
"http",
"hyper",
"hyper-util",
"rustls 0.23.11",
"rustls 0.23.12",
"rustls-pki-types",
"tokio",
"tokio-rustls",
@ -1074,7 +1074,7 @@ checksum = "0122b7114117e64a63ac49f752a5ca4624d534c7b1c7de796ac196381cd2d947"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -1214,13 +1214,14 @@ dependencies = [
[[package]]
name = "mio"
version = "0.8.11"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4"
dependencies = [
"hermit-abi",
"libc",
"wasi",
"windows-sys 0.48.0",
"windows-sys 0.52.0",
]
[[package]]
@ -1323,16 +1324,6 @@ dependencies = [
"libm",
]
[[package]]
name = "num_cpus"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "num_threads"
version = "0.1.7"
@ -1359,9 +1350,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "openssl"
version = "0.10.65"
version = "0.10.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2823eb4c6453ed64055057ea8bd416eda38c71018723869dd043a3b1186115e"
checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
dependencies = [
"bitflags 2.6.0",
"cfg-if",
@ -1380,7 +1371,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -1431,7 +1422,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -1501,7 +1492,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -1888,13 +1879,13 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.11"
version = "0.23.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0"
checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044"
dependencies = [
"once_cell",
"rustls-pki-types",
"rustls-webpki 0.102.5",
"rustls-webpki 0.102.6",
"subtle",
"zeroize",
]
@ -1936,9 +1927,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.102.5"
version = "0.102.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78"
checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e"
dependencies = [
"ring",
"rustls-pki-types",
@ -1986,7 +1977,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -2044,7 +2035,7 @@ dependencies = [
"proc-macro2",
"quote",
"sea-bae",
"syn 2.0.71",
"syn 2.0.72",
"unicode-ident",
]
@ -2108,7 +2099,7 @@ dependencies = [
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
"thiserror",
]
@ -2181,7 +2172,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -2544,11 +2535,10 @@ dependencies = [
[[package]]
name = "sr_download"
version = "0.1.0"
version = "1.0.0"
dependencies = [
"anyhow",
"blake3",
"clap",
"colored",
"futures",
"migration",
@ -2609,9 +2599,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.71"
version = "2.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462"
checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
dependencies = [
"proc-macro2",
"quote",
@ -2627,7 +2617,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -2692,7 +2682,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -2755,32 +2745,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.38.1"
version = "1.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df"
checksum = "9c3318c4fc7126c339a40fbc025927d0328ca32259f68bfe4321660644c1f626"
dependencies = [
"backtrace",
"bytes",
"libc",
"mio",
"num_cpus",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"windows-sys 0.48.0",
"windows-sys 0.52.0",
]
[[package]]
name = "tokio-macros"
version = "2.3.0"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a"
checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -2799,7 +2788,7 @@ version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
dependencies = [
"rustls 0.23.11",
"rustls 0.23.12",
"rustls-pki-types",
"tokio",
]
@ -2870,7 +2859,7 @@ dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"winnow 0.6.14",
"winnow 0.6.15",
]
[[package]]
@ -2920,7 +2909,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]
@ -3112,7 +3101,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
"wasm-bindgen-shared",
]
@ -3146,7 +3135,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@ -3364,9 +3353,9 @@ dependencies = [
[[package]]
name = "winnow"
version = "0.6.14"
version = "0.6.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "374ec40a2d767a3c1b4972d9475ecd557356637be906f2cb3f7fe17a6eb5e22f"
checksum = "557404e450152cd6795bb558bca69e43c585055f4606e3bcae5894fc6dac9ba0"
dependencies = [
"memchr",
]
@ -3407,7 +3396,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn 2.0.72",
]
[[package]]

View File

@ -6,6 +6,7 @@ sqlx_logging = false
[sync]
max_timeout = 1.0
serve_wait_time = 10.0
start_id = 76859
[sync.fast]

View File

@ -1,6 +1,6 @@
[package]
name = "sr_download"
version = "0.1.0"
version = "1.0.0"
edition = "2021"
default-run = "sr_download"
@ -27,4 +27,3 @@ toml = "0.8.15"
blake3 = "1.5.3"
futures = "0.3.30"
colored = "2.1.0"
clap = { version = "4.5.9", features = ["derive"] }

View File

@ -49,12 +49,14 @@ impl Default for FastSyncConfig {
#[serde(rename = "sync")]
pub struct SyncConfig {
pub max_timeout: f32,
pub serve_wait_time: f32,
pub fast: FastSyncConfig,
}
impl Default for SyncConfig {
fn default() -> Self {
Self {
max_timeout: 1.0,
serve_wait_time: 10.0,
fast: FastSyncConfig::default(),
}
}
@ -80,7 +82,11 @@ impl ConfigFile {
Ok(())
}
pub fn timeout_as_duration(&self) -> std::time::Duration {
pub fn serve_duration(&self) -> std::time::Duration {
std::time::Duration::from_secs_f32(self.sync.serve_wait_time)
}
pub fn net_timeout(&self) -> std::time::Duration {
std::time::Duration::from_secs_f32(self.sync.max_timeout)
}

View File

@ -98,6 +98,8 @@ pub async fn find_max_id(db: &DatabaseConnection) -> SaveId {
// 我丢你老母, 有这时间写这个, 我都写完 sql 语句了
let data: Result<Option<i32>, DbErr> = model::main_data::Entity::find()
.order_by_desc(model::main_data::Column::SaveId)
.filter(model::main_data::Column::Len.gt(0))
.filter(model::main_data::Column::SaveType.ne(SaveType::None))
.select_only()
.column(model::main_data::Column::SaveId)
.limit(1)

View File

@ -1,6 +1,6 @@
use colored::Colorize;
use futures::future::select_all;
use std::ops::Range;
use std::{io::Write, ops::Range};
use tokio::sync::oneshot::Receiver;
use tracing::{event, Level};
@ -10,10 +10,9 @@ mod db_part;
mod model;
mod net;
use model::sea_orm_active_enums::SaveType;
use crate::db_part::CoverStrategy;
use migration::SaveId;
use model::sea_orm_active_enums::SaveType;
async fn big_worker(
db: sea_orm::DatabaseConnection,
@ -77,20 +76,101 @@ async fn big_worker(
}
}
async fn main_works(mut stop_receiver: Receiver<()>) -> anyhow::Result<()> {
async fn serve_mode(mut stop_receiver: Receiver<()>) -> anyhow::Result<()> {
let conf = config::ConfigFile::read_or_panic();
let db_connect = db_part::connect(&conf).await?;
db_part::migrate(&db_connect).await?;
let db_max_id = db_part::find_max_id(&db_connect).await;
let mut db_max_id = db_part::find_max_id(&db_connect).await;
event!(
Level::INFO,
"{}",
format!("db max downloaded save_id: {}", db_max_id).green()
format!(
"数据库中最大的现有数据 id 为: {} 将从这里开始下载",
db_max_id
)
.green()
);
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
let serve_wait_time = conf.serve_duration();
let client = net::Downloader::new(conf.net_timeout());
let mut waited = false;
loop {
if stop_receiver.try_recv().is_ok() {
event!(Level::INFO, "{}", "结束下载!".yellow());
// 结束 db
db_connect.close().await?;
return Ok(());
}
let work_id = db_max_id + 1;
match client.try_download_as_any(work_id).await {
Some(file) => {
if waited {
println!();
waited = false;
}
event!(
Level::INFO,
"{}",
format!(
"下载到了新的 {}!(懒得做中文了) ID为: {} 长度: {}",
file.type_name(),
work_id,
file.len()
)
.green()
);
let save_type: SaveType = (&file).into();
match db_part::save_data_to_db(
work_id,
save_type,
file.take_data(),
Some(CoverStrategy::CoverIfDifferent),
&db_connect,
)
.await
{
Ok(_) => {
{
db_max_id = work_id;
event!(
Level::INFO,
"{}",
format!(
"保存好啦! (下一排的每一个 . 代表一个 {:?})",
serve_wait_time
)
.green()
);
};
}
Err(e) => {
event!(Level::ERROR, "呜呜呜, 数据保存失败了: {:?}\n我不玩了!", e);
return Err(e);
}
}
}
None => {
print!(".");
waited = true;
let _ = std::io::stdout().flush();
}
}
tokio::time::sleep(serve_wait_time).await;
}
}
async fn fast_mode(mut stop_receiver: Receiver<()>) -> anyhow::Result<()> {
let conf = config::ConfigFile::read_or_panic();
let db_connect = db_part::connect(&conf).await?;
db_part::migrate(&db_connect).await?;
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
if stop_receiver.try_recv().is_ok() {
event!(Level::INFO, "{}", "Stop download".red());
@ -111,7 +191,7 @@ async fn main_works(mut stop_receiver: Receiver<()>) -> anyhow::Result<()> {
db_connect.close().await?;
return Ok(());
}
let client = net::Downloader::new(conf.timeout_as_duration());
let client = net::Downloader::new(conf.net_timeout());
let end = current_id + worker_size;
works.push(tokio::spawn(big_worker(
db_connect.clone(),
@ -129,7 +209,7 @@ async fn main_works(mut stop_receiver: Receiver<()>) -> anyhow::Result<()> {
return Ok(());
}
while current_id < end_id && works.len() < max_works {
let client = net::Downloader::new(conf.timeout_as_duration());
let client = net::Downloader::new(conf.net_timeout());
let end = current_id + worker_size;
works.push(tokio::spawn(big_worker(
db_connect.clone(),
@ -152,24 +232,39 @@ async fn main() -> anyhow::Result<()> {
tracing_subscriber::fmt().with_max_level(Level::INFO).init();
event!(Level::INFO, "Starting srdownload");
// 初始化一个 ctrl-c 的监听器
let (ctrl_c_sender, ctrl_c_receiver) = tokio::sync::oneshot::channel::<()>();
// 判断是否有 -f / -s 参数
let args: Vec<String> = std::env::args().collect();
let (stop_sender, stop_receiver) = tokio::sync::oneshot::channel::<()>();
// 把 main_works spawn 出去, 这样就可以在主线程检测 ctrl-c 了
let main_works = tokio::spawn(main_works(ctrl_c_receiver));
// ctrl-c 信号处理
let ctrl_c_waiter = tokio::spawn(async move {
tokio::signal::ctrl_c()
.await
.expect("Failed to listen for Ctrl+C event");
event!(Level::INFO, "{}", "Ctrl-C received".red());
ctrl_c_sender.send(()).unwrap();
});
main_works.await??;
if !ctrl_c_waiter.is_finished() {
ctrl_c_waiter.abort()
if args.contains(&"-s".to_string()) {
let job_waiter = tokio::spawn(serve_mode(stop_receiver));
// serve 模式的任务不会结束, 所以需要等待 ctrl-c
tokio::signal::ctrl_c().await?;
let _ = stop_sender.send(()); // 反正不需要管, 发过去了就行
job_waiter.await??;
event!(Level::INFO, "{}", "ctrl-c 收到啦! 停止下载".green());
return Ok(());
} else if args.contains(&"-f".to_string()) {
let stop_waiter = tokio::spawn(async move {
tokio::signal::ctrl_c()
.await
.expect("Failed to listen for Ctrl+C event");
event!(Level::INFO, "{}", "Ctrl-C received".red());
stop_sender.send(()).unwrap();
});
let job_waiter = tokio::spawn(fast_mode(stop_receiver));
// fast 模式的任务会结束, 所以需要等待任务结束
job_waiter.await??;
let _ = stop_waiter.await;
return Ok(());
}
event!(
Level::ERROR,
"{}",
"Please use -s or -f to start the program".red()
);
event!(Level::ERROR, "{}", "Use -s to start serve mode".red());
event!(Level::ERROR, "{}", "Use -f to start fast mode".red());
Ok(())
}