From 0fc6045898a9c91740a86bbb77c6e829769a99e8 Mon Sep 17 00:00:00 2001 From: shenjack <3695888@qq.com> Date: Sat, 20 Jul 2024 19:29:45 +0800 Subject: [PATCH] =?UTF-8?q?=E6=88=91=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=8C?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=80=E4=B8=8B=E8=A1=A8=E5=AE=9A=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 38 +++++++++ .../m20240719_00001_create_main_data_table.rs | 2 +- .../m20240719_00002_create_long_data_table.rs | 2 +- sr_download/Cargo.toml | 3 +- sr_download/src/db.rs | 79 +++++++++++++++++-- sr_download/src/main.rs | 4 + sr_download/src/model/long_data.rs | 2 +- sr_download/src/model/main_data.rs | 2 +- 8 files changed, 122 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index abc5454..891c573 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -135,6 +135,12 @@ dependencies = [ "backtrace", ] +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + [[package]] name = "arrayvec" version = "0.7.4" @@ -266,6 +272,19 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake3" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9ec96fe9a81b5e365f9db71fe00edc4fe4ca2cc7dcb7861f0603012a7caa210" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -422,6 +441,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + [[package]] name = "core-foundation" version = "0.9.4" @@ -1285,6 +1310,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "object" version = "0.36.1" @@ -2490,6 +2524,7 @@ name = "sr_download" version = "0.1.0" dependencies = [ "anyhow", + "blake3", "migration", "reqwest", "sea-orm", @@ -2652,7 +2687,9 @@ checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", + "libc", "num-conv", + "num_threads", "powerfmt", "serde", "time-core", @@ -2894,6 +2931,7 @@ dependencies = [ "sharded-slab", "smallvec", "thread_local", + "time", "tracing", "tracing-core", "tracing-log", diff --git a/migration/src/m20240719_00001_create_main_data_table.rs b/migration/src/m20240719_00001_create_main_data_table.rs index 7afc208..f9a4bd6 100644 --- a/migration/src/m20240719_00001_create_main_data_table.rs +++ b/migration/src/m20240719_00001_create_main_data_table.rs @@ -56,7 +56,7 @@ impl MigrationTrait for Migration { // ad4a4c99162bac6766fa9a658651688c6db4955922f8e5447cb14ad1c1b05825 // len = 64 .col(ColumnDef::new(MainData::BlakeHash).char_len(64).not_null()) - .col(ColumnDef::new(MainData::Len).integer().not_null()) + .col(ColumnDef::new(MainData::Len).big_integer().not_null()) .col(ColumnDef::new(MainData::ShortData).string_len(1024)) .to_owned(), ) diff --git a/migration/src/m20240719_00002_create_long_data_table.rs b/migration/src/m20240719_00002_create_long_data_table.rs index 18f8614..b91eed8 100644 --- a/migration/src/m20240719_00002_create_long_data_table.rs +++ b/migration/src/m20240719_00002_create_long_data_table.rs @@ -19,7 +19,7 @@ impl MigrationTrait for Migration { .not_null() .primary_key(), ) - .col(ColumnDef::new(LongData::Len).string().not_null()) + .col(ColumnDef::new(LongData::Len).big_integer().not_null()) .col(ColumnDef::new(LongData::Text).string().not_null()) .foreign_key( ForeignKey::create() diff --git a/sr_download/Cargo.toml b/sr_download/Cargo.toml index 19afe7b..1655693 100644 --- a/sr_download/Cargo.toml +++ b/sr_download/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" reqwest = "0.12.5" tokio = { version = "1.38.1", features = ["full"] } tracing = "0.1.40" -tracing-subscriber = "0.3.18" +tracing-subscriber = { version = "0.3.18", features = ["local-time"] } anyhow = { version = "1.0.86", features = ["backtrace"] } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -19,3 +19,4 @@ migration = { path = "../migration" } serde = { version = "1.0.204", features = ["serde_derive"] } toml = "0.8.15" +blake3 = "1.5.3" diff --git a/sr_download/src/db.rs b/sr_download/src/db.rs index ec96779..13476a9 100644 --- a/sr_download/src/db.rs +++ b/sr_download/src/db.rs @@ -1,18 +1,87 @@ -use sea_orm::{ConnectOptions, Database, DatabaseConnection}; +use sea_orm::{ + ActiveModelTrait, ConnectOptions, ConnectionTrait, Database, DatabaseConnection, EntityTrait, IntoActiveModel, QueryOrder, QuerySelect +}; +use blake3::Hasher; +use tracing::{event, Level}; -use crate::config::ConfigFile; +use crate::{model, TEXT_DATA_MAX_LEN}; +use crate::{config::ConfigFile, SaveId}; use migration::{Migrator, MigratorTrait}; +use crate::model::sea_orm_active_enums::SaveType; pub async fn connect(conf: &ConfigFile) -> anyhow::Result { let mut opt = ConnectOptions::new(conf.db_url.clone()); opt.max_connections(conf.max_connections) .set_schema_search_path(conf.db_schema.clone()) .sqlx_logging(conf.sqlx_logging); - + event!(Level::INFO, "Connecting to database"); let db: DatabaseConnection = Database::connect(opt).await?; db.ping().await?; - + event!(Level::INFO, "Connected to database, starting migration"); Migrator::up(&db, None).await?; - + event!(Level::INFO, "Migration finished"); Ok(db) } + +pub async fn find_max_id(db: &DatabaseConnection) -> SaveId { + // SELECT save_id from main_data ORDER BY save_id DESC LIMIT 1 + // 我丢你老母, 有这时间写这个, 我都写完 sql 语句了 + match model::main_data::Entity::find() + .order_by_desc(model::main_data::Column::SaveId) + .select_only() + .column(model::main_data::Column::SaveId) + .one(db) + .await + { + Ok(model) => match model { + Some(model) => model.save_id as SaveId, + None => 0, + }, + Err(_) => 0, + } +} + +pub async fn save_ship_to_db( + save_id: SaveId, + data: D, + db: &DatabaseConnection, +) -> anyhow::Result<()> +where + D: Into, +{ + let data: String = data.into(); + // 检查长度 + let data_len = data.len(); + // 计算 blake3 hash + let mut hasher = Hasher::new(); + hasher.update(data.as_bytes()); + let hash = hasher.finalize().to_hex().to_string(); + + if data_len > TEXT_DATA_MAX_LEN { + // 过长, 需要把数据放到 long_data 里 + let new_data = model::main_data::Model { + save_id: save_id as i32, + save_type: SaveType::Ship, + blake_hash: hash, + len: data_len as i32, + short_data: None, + }; + let long_data = model::long_data::Model { + save_id: save_id as i32, + len: data_len as i32, + data, + }; + } else { + // 直接放到 main_data 里即可 + let new_data = model::main_data::Model { + save_id: save_id as i32, + save_type: SaveType::Ship, + blake_hash: hash, + len: data_len as i32, + short_data: Some(data), + }; + new_data.into_active_model().insert(db).await?; + } + + Ok(()) +} diff --git a/sr_download/src/main.rs b/sr_download/src/main.rs index 07d10fa..a11d213 100644 --- a/sr_download/src/main.rs +++ b/sr_download/src/main.rs @@ -7,6 +7,7 @@ mod model; mod net; pub type SaveId = u32; +pub const TEXT_DATA_MAX_LEN: usize = 1024; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -22,6 +23,9 @@ async fn main() -> anyhow::Result<()> { }; let db_connect = db::connect(&conf).await?; + let start_id = db::find_max_id(&db_connect).await; + + event!(Level::INFO, "Starting download from save_id: {}", start_id); Ok(()) } diff --git a/sr_download/src/model/long_data.rs b/sr_download/src/model/long_data.rs index 1956ef5..13bc6cb 100644 --- a/sr_download/src/model/long_data.rs +++ b/sr_download/src/model/long_data.rs @@ -7,7 +7,7 @@ use sea_orm::entity::prelude::*; pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub save_id: i32, - pub len: String, + pub len: i64, pub text: String, } diff --git a/sr_download/src/model/main_data.rs b/sr_download/src/model/main_data.rs index 1fbdfa7..80f78f3 100644 --- a/sr_download/src/model/main_data.rs +++ b/sr_download/src/model/main_data.rs @@ -10,7 +10,7 @@ pub struct Model { pub save_id: i32, pub save_type: SaveType, pub blake_hash: String, - pub len: i32, + pub len: i64, pub short_data: Option, }