This commit is contained in:
shenjack 2024-07-19 18:17:02 +08:00
commit 35d7e7b7bb
Signed by: shenjack
GPG Key ID: 7B1134A979775551
17 changed files with 4313 additions and 0 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/target

2439
Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

6
Cargo.toml Normal file
View File

@ -0,0 +1,6 @@
[workspace]
resolver = "2"
members = [
"sr_download",
"migration"
]

290
get_sr_save.py Normal file
View File

@ -0,0 +1,290 @@
import os
import sys
import time
import requests
import traceback
import threading
# 飞船 API http://jundroo.com/service/SimpleRockets/DownloadRocket?id=
# 存档 API http://jundroo.com/service/SimpleRockets/DownloadSandBox?id=
# (api直接在 text 中返回内容id无效则返回0)
version = "1.0.1"
# 命令行参数:
CLI = f"""
By shenjackyuanjie 3695888@qq.com
SR1 存档下载脚本 版本 {version}
飞船保存目录: ./ship
沙盒保存目录: ./save
命令行参数:
download.py get <id> 下载指定 ID 的飞船/存档
download.py list 列出所有飞船/存档
download.py help 显示帮助信息
download.py version 显示版本信息
download.py serve <id> 间隔一定时间尝试从 <id> 开始下载新的飞船/存档 (默认 60)
参数说明:
-t --time <time> 间隔时间 (默认 10)
download.py gets <start> <end> 下载指定范围的飞船/存档
参数说明:
-nsp --no-ship 不下载飞船
-nsb --no-save 不下载存档
-vv --void-void 无效 ID 也下载
"""
reset_code = '\033[0m'
threaded = True
class DiskStatus:
def __init__(self, save: list[int], ship: list[int], void: list[int] = None):
self.save = save
self.ship = ship
self.void = void or []
def download(id, id_type: int) -> bool:
# 1 ship
# 2 sandbox
url = f"http://jundroo.com/service/SimpleRockets/Download{'Rocket' if id_type == 1 else 'SandBox'}?id={id}"
try:
raw_data = requests.get(url, timeout=10)
except:
# just try again
try:
raw_data = requests.get(url, timeout=10)
except requests.exceptions.RequestException as e:
# 记录当前错误 ID
traceback.print_exc()
with open("error.csv", "a", encoding='utf-8') as f:
f.write(f"{id},{id_type},{e}\n")
return False
if raw_data.status_code != 200:
print(f"ID {id} {'飞船' if id_type == 1 else '沙盒'} 下载失败\n")
return False
raw_data = raw_data.text
if raw_data == "0" or raw_data == "" or len(raw_data) < 1:
return False
with open(f"./{'ship' if id_type == 1 else 'save'}/{id}.xml", "w", encoding='utf-8') as f:
f.write(raw_data)
return True
def download_any(id) -> bool | int:
# 首先尝试飞船
if not download(id, 1):
if not download(id, 2):
# print(f"ID {id} 无效")
with open("void.txt", "a", encoding='utf-8') as f:
f.write(f"{id}\n")
return False
return 2
return 1
def thread_download(id, len, start, end, no_ship, no_save, status: DiskStatus):
color_code = f'\033[3{str(id % 7 + 1)}m'
print(f"{color_code}线程 {threading.current_thread().name}({len}) 开始下载 {start} - {end} {reset_code}")
for i in range(start, end + 1):
# 判定是否已经下载
if i in status.ship or i in status.save or i in status.void:
print(f"{color_code} ID {i} 已存在 {reset_code}")
continue
if no_ship:
if not download(i, 2):
print(f"{color_code} ID {i} 无效-存档 {reset_code}")
elif no_save:
if not download(i, 1):
print(f"{color_code} ID {i} 无效-飞船 {reset_code}")
else:
get_status = download_any(i)
if not get_status:
print(f"{color_code} ID {i} 无效 {reset_code}")
elif get_status == 1:
print(f"{color_code} {time.strftime('[%H:%M:%S]', time.gmtime())} ----------ID {i} 飞船下载完成---------- {reset_code}")
else:
print(f"{color_code} {time.strftime('[%H:%M:%S]', time.gmtime())} ++++++++++ID {i} 存档下载完成++++++++++ {reset_code}")
print(f"{color_code}线程 {threading.current_thread().name} 下载完成 {start} - {end} {reset_code}")
def exit_with_help():
print(CLI)
sys.exit()
def check_disk_status() -> DiskStatus:
ships = []
if not os.path.exists("./ship"):
os.mkdir("./ship")
for file in os.listdir("./ship"):
if file.endswith(".xml"):
ships.append(int(file[:-4]))
saves = []
if not os.path.exists("./save"):
os.mkdir("./save")
for file in os.listdir("./save"):
if file.endswith(".xml"):
saves.append(int(file[:-4]))
voids = []
if os.path.exists("./void.txt"):
with open("./void.txt", "r", encoding='utf-8') as f:
voids = f.readlines()
# 重新写入回去,排序, 去重
voids = [int(i) for i in voids if i != "\n"]
voids = list(set(voids))
# 先检查是否已经存在
# voids = [i for i in voids if i not in ships and i not in saves]
voids.sort()
with open("./void.txt", "w", encoding='utf-8') as f:
for i in voids:
f.write(f"{i}\n")
else:
with open("./void.txt", "x", encoding='utf-8') as f:
...
return DiskStatus(saves, ships, voids)
def download_range():
# 参数检查
if len(sys.argv) < 4:
exit_with_help()
no_ship = False
no_save = False
void_void = False
start = int(sys.argv[2])
end = int(sys.argv[3])
if start > end:
exit_with_help()
if '--void-void' in sys.argv or '-vv' in sys.argv:
print('尝试下载已忽略的 ID')
void_void = True
if "--no-ship" in sys.argv or "-nsp" in sys.argv:
print("不下载飞船")
no_ship = True
if "--no-save" in sys.argv or "-nsb" in sys.argv:
print("错误参数")
exit_with_help()
if "--no-save" in sys.argv or "-nsb" in sys.argv:
print("不下载存档")
no_save = True
print(f"开始下载 {start} - {end}, 不下载飞船: {no_ship}, 不下载存档: {no_save}, 尝试下载已忽略的 ID: {void_void}")
# 多线程下载 每个线程下载 10 个
# 最后一个线程下载剩余的
status = check_disk_status()
# 如果尝试下载已忽略的 ID
# 清空 status.void
if void_void:
status.void = []
thread_pool = []
id = 0
for i in range(start, end + 1, 10):
# 如果这次循环所有存档都已经存在,则跳过
if all((x in status.save or x in status.ship or x in status.void) for x in range(i, min(i+10, end+1))):
print(f"==========ID {i} - {min(i+10, end+1)} 已存在==========")
continue
for thread in thread_pool:
if not thread.is_alive():
thread_pool.pop(thread_pool.index(thread))
if i + 10 > end:
thread_download(id, len(thread_pool), i, end + 1, no_ship, no_save, status)
else:
if threaded:
cache = threading.Thread(target=thread_download, args=(id, len(thread_pool), i, i+10, no_ship, no_save, status, ))
while threading.active_count() >= 48:
time.sleep(0.01)
cache.start()
thread_pool.append(cache)
else:
thread_download(id, len(thread_pool), i, i+10, no_ship, no_save, status)
id += 1
# 开始所有线程
# 等待所有线程结束
for thread in threading.enumerate():
if not thread == threading.current_thread() and not thread == threading.main_thread() and thread.is_alive():
thread.join()
print("下载完成")
check_disk_status()
def serve_download(start_id: int, wait_time: int | None = 60) -> None:
start_status = check_disk_status()
wait_time = wait_time or 10
next_id = start_id
trys = 0
print(f"开始下载 ID {next_id} 之后的飞船和存档,每 {wait_time} 秒检查一次")
while True:
# 检查等候下载的 id 状态
if next_id in start_status.ship or next_id in start_status.save:
print(f"ID {next_id} 已存在")
next_id += 1
continue
status = download_any(next_id)
if status == 1:
print(f" {time.strftime('[%H:%M:%S]', time.gmtime())} ------ ID {next_id} 飞船下载完成 -----")
next_id += 1
trys = 0
continue
elif status == 2:
print(f" {time.strftime('[%H:%M:%S]', time.gmtime())} ++++++ ID {next_id} 存档下载完成 +++++")
next_id += 1
trys = 0
continue
else:
trys += 1
if trys == 1:
print(f"尝试下载 ID {next_id} 次数: 1", end="", flush=True)
else:
back = "\b" * len(str(trys))
print(back, end="")
print(f"{trys}", end="", flush=True)
time.sleep(wait_time)
if __name__ == "__main__":
# 如果直接运行脚本 输出帮助信息
if len(sys.argv) == 1 or sys.argv[1] == "help":
exit_with_help()
# 如果运行脚本时带有参数
if sys.argv[1] == "version":
print(version)
sys.exit()
if sys.argv[1] == "list":
print("飞船列表:")
print(os.listdir("./ship"))
print("存档列表:")
print(os.listdir("./save"))
print(len(sys.argv))
if len(sys.argv) >= 3:
match sys.argv[1]:
case "get":
id = int(sys.argv[2])
status = download_any(id)
if status == 1:
print(f"ID {id} 飞船下载完成")
elif status == 2:
print(f"ID {id} 存档下载完成")
else:
print(f"ID {id} 无效")
sys.exit()
case "gets":
download_range()
case "serve":
id = int(sys.argv[2])
wait_time = None
if '-t' in sys.argv or '--time' in sys.argv:
wait_time = float(sys.argv[sys.argv.index('-t') + 1])
serve_download(id, wait_time)
case _:
print("未知参数", sys.argv)
exit_with_help()

121
main.ps1 Normal file
View File

@ -0,0 +1,121 @@
# 用于批量下载 SR1 存档,然后存储到本地的文件夹中
# 主程序
$save_path = './save'
$ship_path = './ship'
# 创建文件夹
if (!(Test-Path $save_path -PathType Container)) {
New-Item -ItemType Directory -Path $save_path
}
if (!(Test-Path $ship_path -PathType Container)) {
New-Item -ItemType Directory -Path $ship_path
}
# 从命令行输入获取范围
# main.ps1 -s int -e int
# -s int
# -e int
# --no-ship
# --no-save
if ($args.Count -gt 0) {
$range = @()
$no_ship = $false
$no_save = $false
$start = 200709
$end = 300000
for ($i = 0; $i -lt $args.Count; $i++) {
switch ($args[$i]) {
'-s' {
$start = $args[$i + 1]
$i += 1
}
'-e' {
$end = $args[$i + 1]
$i += 1
}
'--no-ship' {
$no_ship = $true
}
'--no-save' {
$no_save = $true
}
}
}
$range = $start..$end
}
# 多线程下载
function Main {
param ($range, $no_ship, $no_save)
if ($no_ship -and $no_save) {
Write-Output "无效参数"
return
}
$range | ForEach-Object -Parallel {
function Download {
param ($Id)
$using:no_save
$using:no_ship
# 飞船 API http://jundroo.com/service/SimpleRockets/DownloadRocket?id=
# 存档 API http://jundroo.com/service/SimpleRockets/DownloadSandBox?id=
# (api直接在Content中返回内容id无效则返回0)
# 先判断 ship 和 save 中有没有已经下载的文件,有则跳过
if (Test-Path "./ship/$Id.xml" -PathType Leaf) {
Write-Host "ID $Id 飞船已存在,跳过"
return
}
if (Test-Path "./save/$Id.xml" -PathType Leaf) {
Write-Host "ID $Id 存档已存在,跳过"
return
}
if ($no_ship) {
$data = Invoke-WebRequest "http://jundroo.com/service/SimpleRockets/DownloadSandBox?id=$Id" -MaximumRetryCount 3
if ($data.Content -eq "0") {
Write-Host "ID $Id 无效,跳过"
}
else {
$data.Content | Out-File "./save/$Id.xml" -Encoding ASCII
Write-Host "-----ID $Id 存档下载成功-----"
}
return
}
$data = Invoke-WebRequest "http://jundroo.com/service/SimpleRockets/DownloadRocket?id=$Id" -MaximumRetryCount 3
if ($data.Content -eq "0") {
Write-Host "ID $Id 非飞船,尝试存档中"
if ($no_save) {
Write-Host "ID $Id 无效,跳过"
return
}
$data = Invoke-WebRequest "http://jundroo.com/service/SimpleRockets/DownloadSandBox?id=$Id"
if ($data.Content -eq "0") {
Write-Host "ID $Id 无效,跳过"
}
else {
$data.Content | Out-File "./save/$Id.xml" -Encoding ASCII
Write-Host "-----ID $Id 存档下载成功-----"
}
}
else {
$data.Content | Out-File "./ship/$Id.xml" -Encoding ASCII
Write-Host "=====ID $Id 飞船下载成功====="
}
}
Download $_
} -ThrottleLimit 10
}
Main $range $no_ship $no_save
Write-Host "下载完成"

19
migration/Cargo.toml Normal file
View File

@ -0,0 +1,19 @@
[package]
name = "migration"
version = "0.1.0"
edition = "2021"
publish = false
[lib]
name = "migration"
path = "src/lib.rs"
[dependencies]
tokio = { version = "1.38", features = ["full"] }
[dependencies.sea-orm-migration]
version = "0.12.15"
features = [
"runtime-tokio-rustls", # `ASYNC_RUNTIME` feature
"sqlx-postgres", # `DATABASE_DRIVER` feature
]

41
migration/README.md Normal file
View File

@ -0,0 +1,41 @@
# Running Migrator CLI
- Generate a new migration file
```sh
cargo run -- generate MIGRATION_NAME
```
- Apply all pending migrations
```sh
cargo run
```
```sh
cargo run -- up
```
- Apply first 10 pending migrations
```sh
cargo run -- up -n 10
```
- Rollback last applied migrations
```sh
cargo run -- down
```
- Rollback last 10 applied migrations
```sh
cargo run -- down -n 10
```
- Drop all tables from the database, then reapply all migrations
```sh
cargo run -- fresh
```
- Rollback all applied migrations, then reapply all migrations
```sh
cargo run -- refresh
```
- Rollback all applied migrations
```sh
cargo run -- reset
```
- Check the status of all migrations
```sh
cargo run -- status
```

16
migration/src/lib.rs Normal file
View File

@ -0,0 +1,16 @@
pub use sea_orm_migration::prelude::*;
mod m20240719_000001_create_main_data_table;
mod m20240719_101428_create_long_data_table;
pub struct Migrator;
#[async_trait::async_trait]
impl MigratorTrait for Migrator {
fn migrations() -> Vec<Box<dyn MigrationTrait>> {
vec![
Box::new(m20240719_000001_create_main_data_table::Migration),
Box::new(m20240719_101428_create_long_data_table::Migration),
]
}
}

View File

@ -0,0 +1,91 @@
use sea_orm::{EnumIter, Iterable};
use sea_orm_migration::prelude::extension::postgres::Type;
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[derive(DeriveIden)]
struct SaveTypeEnum;
#[derive(DeriveIden, EnumIter)]
pub enum SaveTypeVariants {
#[sea_orm(iden = "ship")]
Ship,
#[sea_orm(iden = "save")]
Save,
#[sea_orm(iden = "unknown")]
Unknown,
#[sea_orm(iden = "none")]
None,
}
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.create_type(
Type::create()
.as_enum(SaveTypeEnum)
.values(SaveTypeVariants::iter())
.to_owned(),
)
.await?;
manager
.create_table(
Table::create()
.table(Main::Table)
.if_not_exists()
.col(
ColumnDef::new(Main::SaveId)
.integer()
.not_null()
.primary_key(),
)
.col(
ColumnDef::new(Main::SaveType)
.enumeration(SaveTypeEnum, SaveTypeVariants::iter())
.not_null(),
)
// blake hash:
// ad4a4c99162bac6766fa9a658651688c6db4955922f8e5447cb14ad1c1b05825
// len = 64
.col(ColumnDef::new(Main::BlakeHash).char_len(64).not_null())
.col(ColumnDef::new(Main::Len).integer().not_null())
.col(ColumnDef::new(Main::ShortData).string_len(1024))
.to_owned(),
)
.await
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.drop_table(Table::drop().table(Main::Table).to_owned())
.await
}
}
#[derive(DeriveIden)]
enum Main {
/// 表
Table,
/// 这个存档的 Id
SaveId,
/// 存档类型
/// - ship: 船
/// - save: 存档
/// - unknown: 未知 (没下载呢)
/// - none: 没有存档 (这个 Id 为 空)
SaveType,
/// blake3 hash
/// len = 64
/// 64 位的 blake3 hash
BlakeHash,
/// 存档的长度 (用来过滤太长的存档)
/// 长度 > 1024 的存档会存在隔壁表
Len,
/// 如果长度 < 1024
/// 那就直接存在这
ShortData,
}

View File

@ -0,0 +1,47 @@
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
// Replace the sample below with your own migration scripts
todo!();
manager
.create_table(
Table::create()
.table(Post::Table)
.if_not_exists()
.col(
ColumnDef::new(Post::Id)
.integer()
.not_null()
.auto_increment()
.primary_key(),
)
.col(ColumnDef::new(Post::Title).string().not_null())
.col(ColumnDef::new(Post::Text).string().not_null())
.to_owned(),
)
.await
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
// Replace the sample below with your own migration scripts
todo!();
manager
.drop_table(Table::drop().table(Post::Table).to_owned())
.await
}
}
#[derive(DeriveIden)]
enum Post {
Table,
Id,
Title,
Text,
}

8
migration/src/main.rs Normal file
View File

@ -0,0 +1,8 @@
use sea_orm_migration::prelude::*;
mod m20240719_000001_create_main_data_table;
#[tokio::main]
async fn main() {
cli::run_cli(migration::Migrator).await;
}

55
parse.py Normal file
View File

@ -0,0 +1,55 @@
import traceback
from pathlib import Path
count = 0
num = 0
ship = """<Ship version="1" liftedOff="0" touchingGround="0">
<Parts>
<Part partType="pod-1" id="1" x="0.000000" y="0.750000" angle="0.000000" angleV="0.000000" editorAngle="0">
<Pod throttle="0.000000" name="">
<Staging currentStage="0"/>
</Pod>
</Part>
</Parts>
<Connections/>
</Ship>
"""
def check_ship():
for file in Path("./ship").iterdir():
count += 1
write = False
with open(file, "r", encoding="utf-8") as f:
# print(file.name, end=" ")
try:
first_line = f.readline()
except UnicodeDecodeError:
traceback.print_exc()
continue
if first_line == "empty_ship-v0\n" or first_line == "empty_ship-v0":
print(file, end=" ")
print(count, num)
num += 1
write = True
if write:
with open(file, "w", encoding="utf-8") as f:
f.write(ship)
def remove_save():
save_list = [int(path.name[:-4]) for path in Path("./save").iterdir()]
save_list.sort()
print(save_list)
print("go!")
count = 0
for file in Path("./ship").iterdir():
if int(file.name[:-4]) in save_list:
print(file)
file.unlink()
count += 1
if count >= 50:
print(file.name[:-4])
count = 0
remove_save()

1
sr_download/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/target

1104
sr_download/Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

10
sr_download/Cargo.toml Normal file
View File

@ -0,0 +1,10 @@
[package]
name = "sr_download"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies.reqwest]
version = "0.11.18"
features = ["blocking", "json"]

51
sr_download/src/main.rs Normal file
View File

@ -0,0 +1,51 @@
fn list_file_in_dir(dir: &str, file_vec: &mut Vec<u64>) -> Result<(), Box<dyn std::error::Error>> {
let save_paths = std::fs::read_dir(dir)?;
for save_path in save_paths {
if let Ok(save_path) = save_path {
let file_name = save_path.file_name();
let file_name = file_name.to_str().unwrap();
// 判断文件名是否以 .xml 结尾
if file_name.ends_with(".xml") {
let file_id = file_name.trim_end_matches(".xml");
let file_id = file_id.parse::<u64>()?;
file_vec.push(file_id);
}
}
}
file_vec.sort();
Ok(())
}
enum FileKind {
Ship,
Save,
}
/// 飞船 API http://jundroo.com/service/SimpleRockets/DownloadRocket?id=
/// 存档 API http://jundroo.com/service/SimpleRockets/DownloadSandBox?id=
/// curl http://jundroo.com/service/SimpleRockets/DownloadRocket?id=144444
fn get_file_from_jundroo(id: u64) -> Result<FileKind, Box<dyn std::error::Error>> {
let ship_try = reqwest::blocking::get(format!(
"http://jundroo.com/service/SimpleRockets/DownloadRocket?id={}",
id
))?;
println!("ship_try: {:?}", ship_try);
println!("body: {}", ship_try.text()?);
Ok(FileKind::Ship)
}
fn main() {
let mut ship_vec: Vec<u64> = Vec::with_capacity(82_0000);
let mut save_vec: Vec<u64> = Vec::with_capacity(29_0000);
list_file_in_dir("./ship", &mut ship_vec).unwrap();
list_file_in_dir("./save", &mut save_vec).unwrap();
let _ = get_file_from_jundroo(144444);
for i in 144444..144450 {
let _ = get_file_from_jundroo(i);
}
println!("ship: {}, save: {}", ship_vec.len(), save_vec.len());
}

13
sr_download/src/net.rs Normal file
View File

@ -0,0 +1,13 @@
use Request::blocking::Client;
pub struct Downloader {
pub client: Client
}
impl Downloader {
pub fn new() -> Self {
Self {
client: Client::new()
}
}
}