diff --git a/Cargo.lock b/Cargo.lock index e57cd2d..b127814 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,9 +182,9 @@ dependencies = [ name = "furigana_gen" version = "0.1.0" dependencies = [ + "lz4_flex", "once_cell", "regex", - "ruzstd", "vibrato", ] @@ -256,6 +256,15 @@ dependencies = [ "libc", ] +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +dependencies = [ + "twox-hash", +] + [[package]] name = "memchr" version = "2.7.4" @@ -441,16 +450,6 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" -[[package]] -name = "ruzstd" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c8b8f3d26bd9f945e5cbae77f7cdfbf37af9a66956f1115eb4516e45df519f4" -dependencies = [ - "byteorder", - "twox-hash", -] - [[package]] name = "ryu" version = "1.0.18" diff --git a/Cargo.toml b/Cargo.toml index 8db4433..13ffae0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,6 @@ edition = "2021" [dependencies] vibrato = "0.5" -ruzstd = "0.7" +lz4_flex = "0.11" regex = "1.10" once_cell = "1.19" diff --git a/dictionary/system.dic b/dictionary/system.dic new file mode 100644 index 0000000..f2186f4 Binary files /dev/null and b/dictionary/system.dic differ diff --git a/dictionary/system.dic.lz4 b/dictionary/system.dic.lz4 new file mode 100644 index 0000000..af5769d Binary files /dev/null and b/dictionary/system.dic.lz4 differ diff --git a/dictionary/system.dic.xz b/dictionary/system.dic.xz new file mode 100644 index 0000000..7cb1e03 Binary files /dev/null and b/dictionary/system.dic.xz differ diff --git a/dictionary/system.dic.zst b/dictionary/system.dic.zst deleted file mode 100644 index 7a24093..0000000 Binary files a/dictionary/system.dic.zst and /dev/null differ diff --git a/src/main.rs b/src/main.rs index 5e7bbbe..f8365cb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,17 +3,23 @@ use std::{ io::{Cursor, Read}, }; +use lz4_flex::frame::FrameDecoder; use once_cell::sync::Lazy; use regex::Regex; -use ruzstd::StreamingDecoder; use vibrato::{Dictionary, Tokenizer}; -const DICT: &[u8] = include_bytes!("../dictionary/system.dic.zst"); +const DICT: &[u8] = include_bytes!("../dictionary/system.dic.lz4"); fn main() { let dict = { - let decoder = StreamingDecoder::new(Cursor::new(DICT)).unwrap(); - Dictionary::read(decoder).unwrap() + // Note: we could just pass the decoder straight to `Dictionary::read()` + // below, and it would work. However, that ends up being slower than + // first decompressing the whole thing ahead of time. + let mut decoder = FrameDecoder::new(Cursor::new(DICT)); + let mut data = Vec::new(); + decoder.read_to_end(&mut data).unwrap(); + + Dictionary::read(Cursor::new(&data)).unwrap() }; let text = {