From 199d09a292e5416387a36c36a99e560ae0df6aca Mon Sep 17 00:00:00 2001 From: shenjack <3695888@qq.com> Date: Wed, 24 Jul 2024 00:19:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A5=BD=E6=AC=B8=EF=BC=81=E5=87=BA=EF=BC=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 111 +++++++++++++--------------- config_template.toml | 1 + sr_download/Cargo.toml | 3 +- sr_download/src/config.rs | 8 +- sr_download/src/db_part.rs | 2 + sr_download/src/main.rs | 147 ++++++++++++++++++++++++++++++------- 6 files changed, 182 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1631af8..7acffbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -137,9 +137,9 @@ dependencies = [ [[package]] name = "arrayref" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" [[package]] name = "arrayvec" @@ -166,7 +166,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -177,7 +177,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -314,7 +314,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", "syn_derive", ] @@ -391,9 +391,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.9" +version = "4.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" +checksum = "8f6b81fb3c84f5563d509c59b5a48d935f689e993afa90fe39047f05adef9142" dependencies = [ "clap_builder", "clap_derive", @@ -401,9 +401,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.9" +version = "4.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" +checksum = "5ca6706fd5224857d9ac5eb9355f6683563cc0541c7cd9d014043b57cbec78ac" dependencies = [ "anstream", "anstyle", @@ -420,7 +420,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -743,7 +743,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -980,7 +980,7 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls 0.23.11", + "rustls 0.23.12", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1074,7 +1074,7 @@ checksum = "0122b7114117e64a63ac49f752a5ca4624d534c7b1c7de796ac196381cd2d947" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1214,13 +1214,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ + "hermit-abi", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1323,16 +1324,6 @@ dependencies = [ "libm", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "num_threads" version = "0.1.7" @@ -1359,9 +1350,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" -version = "0.10.65" +version = "0.10.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2823eb4c6453ed64055057ea8bd416eda38c71018723869dd043a3b1186115e" +checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -1380,7 +1371,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1431,7 +1422,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1501,7 +1492,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1888,13 +1879,13 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.11" +version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ "once_cell", "rustls-pki-types", - "rustls-webpki 0.102.5", + "rustls-webpki 0.102.6", "subtle", "zeroize", ] @@ -1936,9 +1927,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.5" +version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ "ring", "rustls-pki-types", @@ -1986,7 +1977,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -2044,7 +2035,7 @@ dependencies = [ "proc-macro2", "quote", "sea-bae", - "syn 2.0.71", + "syn 2.0.72", "unicode-ident", ] @@ -2108,7 +2099,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", "thiserror", ] @@ -2181,7 +2172,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -2544,11 +2535,10 @@ dependencies = [ [[package]] name = "sr_download" -version = "0.1.0" +version = "1.0.0" dependencies = [ "anyhow", "blake3", - "clap", "colored", "futures", "migration", @@ -2609,9 +2599,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.71" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -2627,7 +2617,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -2692,7 +2682,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -2755,32 +2745,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.1" +version = "1.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" +checksum = "9c3318c4fc7126c339a40fbc025927d0328ca32259f68bfe4321660644c1f626" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -2799,7 +2788,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.11", + "rustls 0.23.12", "rustls-pki-types", "tokio", ] @@ -2870,7 +2859,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.14", + "winnow 0.6.15", ] [[package]] @@ -2920,7 +2909,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -3112,7 +3101,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", "wasm-bindgen-shared", ] @@ -3146,7 +3135,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3364,9 +3353,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.14" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374ec40a2d767a3c1b4972d9475ecd557356637be906f2cb3f7fe17a6eb5e22f" +checksum = "557404e450152cd6795bb558bca69e43c585055f4606e3bcae5894fc6dac9ba0" dependencies = [ "memchr", ] @@ -3407,7 +3396,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] diff --git a/config_template.toml b/config_template.toml index 9c95eea..55254cd 100644 --- a/config_template.toml +++ b/config_template.toml @@ -6,6 +6,7 @@ sqlx_logging = false [sync] max_timeout = 1.0 +serve_wait_time = 10.0 start_id = 76859 [sync.fast] diff --git a/sr_download/Cargo.toml b/sr_download/Cargo.toml index 7149a7f..e55aa68 100644 --- a/sr_download/Cargo.toml +++ b/sr_download/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sr_download" -version = "0.1.0" +version = "1.0.0" edition = "2021" default-run = "sr_download" @@ -27,4 +27,3 @@ toml = "0.8.15" blake3 = "1.5.3" futures = "0.3.30" colored = "2.1.0" -clap = { version = "4.5.9", features = ["derive"] } diff --git a/sr_download/src/config.rs b/sr_download/src/config.rs index b55ba2d..8c97a32 100644 --- a/sr_download/src/config.rs +++ b/sr_download/src/config.rs @@ -49,12 +49,14 @@ impl Default for FastSyncConfig { #[serde(rename = "sync")] pub struct SyncConfig { pub max_timeout: f32, + pub serve_wait_time: f32, pub fast: FastSyncConfig, } impl Default for SyncConfig { fn default() -> Self { Self { max_timeout: 1.0, + serve_wait_time: 10.0, fast: FastSyncConfig::default(), } } @@ -80,7 +82,11 @@ impl ConfigFile { Ok(()) } - pub fn timeout_as_duration(&self) -> std::time::Duration { + pub fn serve_duration(&self) -> std::time::Duration { + std::time::Duration::from_secs_f32(self.sync.serve_wait_time) + } + + pub fn net_timeout(&self) -> std::time::Duration { std::time::Duration::from_secs_f32(self.sync.max_timeout) } diff --git a/sr_download/src/db_part.rs b/sr_download/src/db_part.rs index a9ba53e..cfe49ca 100644 --- a/sr_download/src/db_part.rs +++ b/sr_download/src/db_part.rs @@ -98,6 +98,8 @@ pub async fn find_max_id(db: &DatabaseConnection) -> SaveId { // 我丢你老母, 有这时间写这个, 我都写完 sql 语句了 let data: Result, DbErr> = model::main_data::Entity::find() .order_by_desc(model::main_data::Column::SaveId) + .filter(model::main_data::Column::Len.gt(0)) + .filter(model::main_data::Column::SaveType.ne(SaveType::None)) .select_only() .column(model::main_data::Column::SaveId) .limit(1) diff --git a/sr_download/src/main.rs b/sr_download/src/main.rs index 69b933b..4873b38 100644 --- a/sr_download/src/main.rs +++ b/sr_download/src/main.rs @@ -1,6 +1,6 @@ use colored::Colorize; use futures::future::select_all; -use std::ops::Range; +use std::{io::Write, ops::Range}; use tokio::sync::oneshot::Receiver; use tracing::{event, Level}; @@ -10,10 +10,9 @@ mod db_part; mod model; mod net; -use model::sea_orm_active_enums::SaveType; - use crate::db_part::CoverStrategy; use migration::SaveId; +use model::sea_orm_active_enums::SaveType; async fn big_worker( db: sea_orm::DatabaseConnection, @@ -77,20 +76,101 @@ async fn big_worker( } } -async fn main_works(mut stop_receiver: Receiver<()>) -> anyhow::Result<()> { +async fn serve_mode(mut stop_receiver: Receiver<()>) -> anyhow::Result<()> { let conf = config::ConfigFile::read_or_panic(); let db_connect = db_part::connect(&conf).await?; db_part::migrate(&db_connect).await?; - let db_max_id = db_part::find_max_id(&db_connect).await; + let mut db_max_id = db_part::find_max_id(&db_connect).await; event!( Level::INFO, "{}", - format!("db max downloaded save_id: {}", db_max_id).green() + format!( + "数据库中最大的现有数据 id 为: {} 将从这里开始下载", + db_max_id + ) + .green() ); - tokio::time::sleep(std::time::Duration::from_secs(3)).await; + let serve_wait_time = conf.serve_duration(); + let client = net::Downloader::new(conf.net_timeout()); + + let mut waited = false; + loop { + if stop_receiver.try_recv().is_ok() { + event!(Level::INFO, "{}", "结束下载!".yellow()); + // 结束 db + db_connect.close().await?; + return Ok(()); + } + + let work_id = db_max_id + 1; + match client.try_download_as_any(work_id).await { + Some(file) => { + if waited { + println!(); + waited = false; + } + event!( + Level::INFO, + "{}", + format!( + "下载到了新的 {}!(懒得做中文了) ID为: {} 长度: {}", + file.type_name(), + work_id, + file.len() + ) + .green() + ); + let save_type: SaveType = (&file).into(); + match db_part::save_data_to_db( + work_id, + save_type, + file.take_data(), + Some(CoverStrategy::CoverIfDifferent), + &db_connect, + ) + .await + { + Ok(_) => { + { + db_max_id = work_id; + event!( + Level::INFO, + "{}", + format!( + "保存好啦! (下一排的每一个 . 代表一个 {:?})", + serve_wait_time + ) + .green() + ); + }; + } + Err(e) => { + event!(Level::ERROR, "呜呜呜, 数据保存失败了: {:?}\n我不玩了!", e); + return Err(e); + } + } + } + None => { + print!("."); + waited = true; + let _ = std::io::stdout().flush(); + } + } + + tokio::time::sleep(serve_wait_time).await; + } +} + +async fn fast_mode(mut stop_receiver: Receiver<()>) -> anyhow::Result<()> { + let conf = config::ConfigFile::read_or_panic(); + + let db_connect = db_part::connect(&conf).await?; + db_part::migrate(&db_connect).await?; + + tokio::time::sleep(std::time::Duration::from_secs(2)).await; if stop_receiver.try_recv().is_ok() { event!(Level::INFO, "{}", "Stop download".red()); @@ -111,7 +191,7 @@ async fn main_works(mut stop_receiver: Receiver<()>) -> anyhow::Result<()> { db_connect.close().await?; return Ok(()); } - let client = net::Downloader::new(conf.timeout_as_duration()); + let client = net::Downloader::new(conf.net_timeout()); let end = current_id + worker_size; works.push(tokio::spawn(big_worker( db_connect.clone(), @@ -129,7 +209,7 @@ async fn main_works(mut stop_receiver: Receiver<()>) -> anyhow::Result<()> { return Ok(()); } while current_id < end_id && works.len() < max_works { - let client = net::Downloader::new(conf.timeout_as_duration()); + let client = net::Downloader::new(conf.net_timeout()); let end = current_id + worker_size; works.push(tokio::spawn(big_worker( db_connect.clone(), @@ -152,24 +232,39 @@ async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt().with_max_level(Level::INFO).init(); event!(Level::INFO, "Starting srdownload"); - // 初始化一个 ctrl-c 的监听器 - let (ctrl_c_sender, ctrl_c_receiver) = tokio::sync::oneshot::channel::<()>(); + // 判断是否有 -f / -s 参数 + let args: Vec = std::env::args().collect(); + let (stop_sender, stop_receiver) = tokio::sync::oneshot::channel::<()>(); - // 把 main_works spawn 出去, 这样就可以在主线程检测 ctrl-c 了 - let main_works = tokio::spawn(main_works(ctrl_c_receiver)); - - // ctrl-c 信号处理 - let ctrl_c_waiter = tokio::spawn(async move { - tokio::signal::ctrl_c() - .await - .expect("Failed to listen for Ctrl+C event"); - event!(Level::INFO, "{}", "Ctrl-C received".red()); - ctrl_c_sender.send(()).unwrap(); - }); - - main_works.await??; - if !ctrl_c_waiter.is_finished() { - ctrl_c_waiter.abort() + if args.contains(&"-s".to_string()) { + let job_waiter = tokio::spawn(serve_mode(stop_receiver)); + // serve 模式的任务不会结束, 所以需要等待 ctrl-c + tokio::signal::ctrl_c().await?; + let _ = stop_sender.send(()); // 反正不需要管, 发过去了就行 + job_waiter.await??; + event!(Level::INFO, "{}", "ctrl-c 收到啦! 停止下载".green()); + return Ok(()); + } else if args.contains(&"-f".to_string()) { + let stop_waiter = tokio::spawn(async move { + tokio::signal::ctrl_c() + .await + .expect("Failed to listen for Ctrl+C event"); + event!(Level::INFO, "{}", "Ctrl-C received".red()); + stop_sender.send(()).unwrap(); + }); + let job_waiter = tokio::spawn(fast_mode(stop_receiver)); + // fast 模式的任务会结束, 所以需要等待任务结束 + job_waiter.await??; + let _ = stop_waiter.await; + return Ok(()); } + + event!( + Level::ERROR, + "{}", + "Please use -s or -f to start the program".red() + ); + event!(Level::ERROR, "{}", "Use -s to start serve mode".red()); + event!(Level::ERROR, "{}", "Use -f to start fast mode".red()); Ok(()) }