我的问题，修改一下表定义

Cargo.lock

@@ -135,6 +135,12 @@ dependencies = [
  "backtrace",
 ]
 
+[[package]]
+name = "arrayref"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545"
+
 [[package]]
 name = "arrayvec"
 version = "0.7.4"
@@ -266,6 +272,19 @@ dependencies = [
  "wyz",
 ]
 
+[[package]]
+name = "blake3"
+version = "1.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9ec96fe9a81b5e365f9db71fe00edc4fe4ca2cc7dcb7861f0603012a7caa210"
+dependencies = [
+ "arrayref",
+ "arrayvec",
+ "cc",
+ "cfg-if",
+ "constant_time_eq",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -422,6 +441,12 @@ version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
 
+[[package]]
+name = "constant_time_eq"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
+
 [[package]]
 name = "core-foundation"
 version = "0.9.4"
@@ -1285,6 +1310,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "num_threads"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "object"
 version = "0.36.1"
@@ -2490,6 +2524,7 @@ name = "sr_download"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "blake3",
  "migration",
  "reqwest",
  "sea-orm",
@@ -2652,7 +2687,9 @@ checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
 dependencies = [
  "deranged",
  "itoa",
+ "libc",
  "num-conv",
+ "num_threads",
  "powerfmt",
  "serde",
  "time-core",
@@ -2894,6 +2931,7 @@ dependencies = [
  "sharded-slab",
  "smallvec",
  "thread_local",
+ "time",
  "tracing",
  "tracing-core",
  "tracing-log",

migration/src/m20240719_00001_create_main_data_table.rs

@@ -56,7 +56,7 @@ impl MigrationTrait for Migration {
                     // ad4a4c99162bac6766fa9a658651688c6db4955922f8e5447cb14ad1c1b05825
                     // len = 64
                     .col(ColumnDef::new(MainData::BlakeHash).char_len(64).not_null())
-                    .col(ColumnDef::new(MainData::Len).integer().not_null())
+                    .col(ColumnDef::new(MainData::Len).big_integer().not_null())
                     .col(ColumnDef::new(MainData::ShortData).string_len(1024))
                     .to_owned(),
             )

migration/src/m20240719_00002_create_long_data_table.rs

@@ -19,7 +19,7 @@ impl MigrationTrait for Migration {
                             .not_null()
                             .primary_key(),
                     )
-                    .col(ColumnDef::new(LongData::Len).string().not_null())
+                    .col(ColumnDef::new(LongData::Len).big_integer().not_null())
                     .col(ColumnDef::new(LongData::Text).string().not_null())
                     .foreign_key(
                         ForeignKey::create()

sr_download/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 reqwest = "0.12.5"
 tokio = { version = "1.38.1", features = ["full"] }
 tracing = "0.1.40"
-tracing-subscriber = "0.3.18"
+tracing-subscriber = { version = "0.3.18", features = ["local-time"] }
 anyhow = { version = "1.0.86", features = ["backtrace"] }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -19,3 +19,4 @@ migration = { path = "../migration" }
 
 serde = { version = "1.0.204", features = ["serde_derive"] }
 toml = "0.8.15"
+blake3 = "1.5.3"

sr_download/src/db.rs

@@ -1,18 +1,87 @@
-use sea_orm::{ConnectOptions, Database, DatabaseConnection};
+use sea_orm::{
+    ActiveModelTrait, ConnectOptions, ConnectionTrait, Database, DatabaseConnection, EntityTrait, IntoActiveModel, QueryOrder, QuerySelect
+};
+use blake3::Hasher;
+use tracing::{event, Level};
 
-use crate::config::ConfigFile;
+use crate::{model, TEXT_DATA_MAX_LEN};
+use crate::{config::ConfigFile, SaveId};
 use migration::{Migrator, MigratorTrait};
+use crate::model::sea_orm_active_enums::SaveType;
 
 pub async fn connect(conf: &ConfigFile) -> anyhow::Result<DatabaseConnection> {
     let mut opt = ConnectOptions::new(conf.db_url.clone());
     opt.max_connections(conf.max_connections)
         .set_schema_search_path(conf.db_schema.clone())
         .sqlx_logging(conf.sqlx_logging);
-
+    event!(Level::INFO, "Connecting to database");
     let db: DatabaseConnection = Database::connect(opt).await?;
     db.ping().await?;
-
+    event!(Level::INFO, "Connected to database, starting migration");
     Migrator::up(&db, None).await?;
-
+    event!(Level::INFO, "Migration finished");
     Ok(db)
 }
+
+pub async fn find_max_id(db: &DatabaseConnection) -> SaveId {
+    // SELECT save_id from main_data ORDER BY save_id DESC LIMIT 1
+    // 我丢你老母, 有这时间写这个, 我都写完 sql 语句了
+    match model::main_data::Entity::find()
+        .order_by_desc(model::main_data::Column::SaveId)
+        .select_only()
+        .column(model::main_data::Column::SaveId)
+        .one(db)
+        .await
+    {
+        Ok(model) => match model {
+            Some(model) => model.save_id as SaveId,
+            None => 0,
+        },
+        Err(_) => 0,
+    }
+}
+
+pub async fn save_ship_to_db<D>(
+    save_id: SaveId,
+    data: D,
+    db: &DatabaseConnection,
+) -> anyhow::Result<()>
+where
+    D: Into<String>,
+{
+    let data: String = data.into();
+    // 检查长度
+    let data_len = data.len();
+    // 计算 blake3 hash
+    let mut hasher = Hasher::new();
+    hasher.update(data.as_bytes());
+    let hash = hasher.finalize().to_hex().to_string();
+
+    if data_len > TEXT_DATA_MAX_LEN {
+        // 过长, 需要把数据放到 long_data 里
+        let new_data = model::main_data::Model {
+            save_id: save_id as i32,
+            save_type: SaveType::Ship,
+            blake_hash: hash,
+            len: data_len as i32,
+            short_data: None,
+        };
+        let long_data = model::long_data::Model {
+            save_id: save_id as i32,
+            len: data_len as i32,
+            data,
+        };
+    } else {
+        // 直接放到 main_data 里即可
+        let new_data = model::main_data::Model {
+            save_id: save_id as i32,
+            save_type: SaveType::Ship,
+            blake_hash: hash,
+            len: data_len as i32,
+            short_data: Some(data),
+        };
+        new_data.into_active_model().insert(db).await?;
+    }
+
+    Ok(())
+}

sr_download/src/main.rs

@@ -7,6 +7,7 @@ mod model;
 mod net;
 
 pub type SaveId = u32;
+pub const TEXT_DATA_MAX_LEN: usize = 1024;
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
@@ -22,6 +23,9 @@ async fn main() -> anyhow::Result<()> {
     };
 
     let db_connect = db::connect(&conf).await?;
+    let start_id = db::find_max_id(&db_connect).await;
+
+    event!(Level::INFO, "Starting download from save_id: {}", start_id);
 
     Ok(())
 }

sr_download/src/model/long_data.rs

@@ -7,7 +7,7 @@ use sea_orm::entity::prelude::*;
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
     pub save_id: i32,
-    pub len: String,
+    pub len: i64,
     pub text: String,
 }
 

sr_download/src/model/main_data.rs

@@ -10,7 +10,7 @@ pub struct Model {
     pub save_id: i32,
     pub save_type: SaveType,
     pub blake_hash: String,
-    pub len: i32,
+    pub len: i64,
     pub short_data: Option<String>,
 }