chore(代码变更): 1. 清理无用代码 2. 更新依赖库 3. 优化项目结构 4. 修复小型bug 5. 增加注释以提高可读性
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
[package]
|
||||
name = "flex-backend"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
minicpm-inference = { path = "../../crates/minicpm-inference" }
|
||||
burn = { version = "0.21", default-features = false, features = ["std", "flex"] }
|
||||
anyhow = "1.0"
|
||||
@@ -0,0 +1,82 @@
|
||||
// Flex 后端推理示例 —— 适用于 Tauri / WebAssembly 环境
|
||||
//
|
||||
// burn-flex 是纯 Rust CPU 后端,无 GPU 依赖,完美适配 Tauri 应用。
|
||||
// 支持 SIMD 加速(AVX2/NEON)和多线程(rayon)。
|
||||
//
|
||||
// 运行方式:cargo run --release -p flex-backend [--full|--half|--q8]
|
||||
|
||||
use burn::backend::{flex::FlexDevice, Flex};
|
||||
use minicpm_inference::{GenerationConfig, MiniCPM};
|
||||
use std::io::Write;
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let mode = args.get(1).map(|s| s.as_str()).unwrap_or("--full");
|
||||
|
||||
let (model_dir, config_path, tokenizer_path, is_q8) = match mode {
|
||||
"--half" => (
|
||||
"model_half",
|
||||
"MiniCPM5-1B/config.json",
|
||||
"MiniCPM5-1B/tokenizer.json",
|
||||
false,
|
||||
),
|
||||
"--q8" => (
|
||||
"model_q8",
|
||||
"MiniCPM5-1B/config.json",
|
||||
"MiniCPM5-1B/tokenizer.json",
|
||||
true,
|
||||
),
|
||||
_ => (
|
||||
"model",
|
||||
"MiniCPM5-1B/config.json",
|
||||
"MiniCPM5-1B/tokenizer.json",
|
||||
false,
|
||||
),
|
||||
};
|
||||
|
||||
println!("正在加载模型(后端:Flex,模式:{mode})...");
|
||||
println!("Flex 后端特性:纯 Rust、SIMD 加速、多线程");
|
||||
let start = Instant::now();
|
||||
|
||||
let device = FlexDevice::default();
|
||||
let model: MiniCPM<Flex> = if is_q8 {
|
||||
MiniCPM::<Flex>::load_q8(model_dir, config_path, tokenizer_path, &device)?
|
||||
} else {
|
||||
let model_path = format!("{model_dir}/model");
|
||||
MiniCPM::<Flex>::load(&model_path, config_path, tokenizer_path, &device)?
|
||||
};
|
||||
|
||||
println!("模型加载完成,耗时: {:.2?}", start.elapsed());
|
||||
|
||||
let prompt = "需要处理生成一个河南的旅游计划,要求如下:
|
||||
1. 旅游计划为三天两夜的行程安排。
|
||||
2. 每天的行程安排包括景点、餐饮和住宿建议。
|
||||
3. 景点安排要考虑交通便利性和时间合理性。
|
||||
请生成详细的旅游计划,包含每天的行程安排、景点介绍、餐饮推荐和住宿建议。
|
||||
";
|
||||
println!("\n用户: {prompt}");
|
||||
print!("Assistant: ");
|
||||
std::io::stdout().flush()?;
|
||||
|
||||
let config = GenerationConfig {
|
||||
max_new_tokens: Some(1200),
|
||||
temperature: 0.7,
|
||||
top_p: 0.95,
|
||||
};
|
||||
|
||||
let start = Instant::now();
|
||||
let stream = model.generate_stream(prompt, false, &config)?;
|
||||
|
||||
for text in stream {
|
||||
print!("{text}");
|
||||
std::io::stdout().flush().ok();
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
println!();
|
||||
println!("\n生成耗时: {:.2?}", elapsed);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user