https://github.com/blackglory/ultra-nlp
🌿 A NLP library.
https://github.com/blackglory/ultra-nlp
crate library rust
Last synced: 3 months ago
JSON representation
🌿 A NLP library.
- Host: GitHub
- URL: https://github.com/blackglory/ultra-nlp
- Owner: BlackGlory
- License: mit
- Created: 2022-06-28T06:07:14.000Z (about 4 years ago)
- Default Branch: master
- Last Pushed: 2024-03-03T17:34:35.000Z (over 2 years ago)
- Last Synced: 2025-02-03T15:53:03.632Z (over 1 year ago)
- Topics: crate, library, rust
- Language: Rust
- Homepage: https://crates.io/crates/ultra-nlp
- Size: 216 KB
- Stars: 1
- Watchers: 3
- Forks: 0
- Open Issues: 0
-
Metadata Files:
- Readme: README.md
- Changelog: CHANGELOG.md
- License: LICENSE
Awesome Lists containing this project
README
# ultra-nlp
## Install
```sh
cargo add ultra-nlp
```
## Usage
### ngrams
```rs
let text = "你好世界";
let result = ngrams(text, 2);
assert_eq!(
result
.into_iter()
.collect::>(),
vec!["你好", "好世", "世界"]
);
```
### extract_consecutive_chinese_chars
```rs
let text = "foo中文bar字符baz";
let result = extract_consecutive_chinese_chars(text);
assert_eq!(
result
.into_iter()
.collect::>(),
vec!["中文", "字符"]
);
```
### extract_consecutive_letters
```rs
let text = "foo中文，bar,字符baz";
let result = extract_consecutive_letters(text);
assert_eq!(
result
.into_iter()
.collect::>(),
vec!["foo中文", "bar", "字符baz"]
);
```
### cedarwood(slow, low memory usage)
#### Ingore unmatched contents
```rs
use ultra_nlp::BehaviorForUnmatched,
use ultra_nlp::cedarwood::{
segment_fully,
ForwardDictionary,
};
let text = " 南京市长江大桥, hello world ";
let dict = ForwardDictionary::new(
vec!["南京", "南京市", "市长", "长江", "大桥", "你好世界"]
).unwrap();
let result = segment_fully(
text,
&dict,
BehaviorForUnmatched::Ignore
);
assert_eq!(
result
.iter()
.map(|x| x.range().extract(text))
.collect::>(),
vec!["南京", "南京市", "市长", "长江", "大桥"]
);
```
#### Keep unmatched contents as chars
```rs
use ultra_nlp::BehaviorForUnmatched,
use ultra_nlp::cedarwood::{
segment_fully,
ForwardDictionary,
};
let text = " 南京市长江大桥, hello world ";
let dict = ForwardDictionary::new(
vec!["南京", "南京市", "市长", "长江", "大桥", "你好世界"]
).unwrap();
let result = segment_fully(
text,
&dict,
BehaviorForUnmatched::KeepAsChars
);
assert_eq!(
result
.iter()
.map(|x| x.range().extract(text))
.collect::>(),
vec![
" ", "南京", "南京市", "市长", "长江", "大桥", ",", " ", "h", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d", " ",
]
);
```
#### Keep unmatched ocntents as words
```rs
use ultra_nlp::BehaviorForUnmatched,
use ultra_nlp::cedarwood::{
segment_fully,
ForwardDictionary,
};
let text = " 南京市长江大桥, hello world ";
let dict = ForwardDictionary::new(
vec!["南京", "南京市", "市长", "长江", "大桥", "你好世界"]
).unwrap();
let result = segment_fully(
text,
&dict,
BehaviorForUnmatched::KeepAsWords
);
assert_eq!(
result
.iter()
.map(|x| x.range().extract(text))
.collect::>(),
vec![
" ", "南京", "南京市", "市长", "长江", "大桥", ", hello world ",
]
);
```
### daachorse(fast, high memory usage)
#### Ignore unmatched contents
```rs
use ultra_nlp::BehaviorForUnmatched,
use ultra_nlp::daachorse::{
segment_fully,
StandardDictionary,
};
let text = " 南京市长江大桥, hello world ";
let dict = StandardDictionary::new(
vec!["南京", "南京市", "市长", "长江", "大桥", "你好世界"]
).unwrap();
let result = segment_fully(text, &dict, BehaviorForUnmatched::Ignore);
assert_eq!(
result
.iter()
.map(|x| x.range().extract(text))
.collect::>(),
vec![
"南京", "南京市", "市长", "长江", "大桥",
]
);
```
#### Keep unmatched contents as chars
```rs
use ultra_nlp::BehaviorForUnmatched,
use ultra_nlp::daachorse::{
segment_fully,
StandardDictionary,
};
let text = " 南京市长江大桥, hello world ";
let dict = StandardDictionary::new(
vec!["南京", "南京市", "市长", "长江", "大桥", "你好世界"]
).unwrap();
let result = segment_fully(text, &dict, BehaviorForUnmatched::KeepAsChars);
assert_eq!(
result
.iter()
.map(|x| x.range().extract(text))
.collect::>(),
vec![
" ", "南京", "南京市", "市长", "长江", "大桥", ",", " ", "h", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d", " ",
]
);
```
#### Keep unmatched contents as words
```rs
use ultra_nlp::BehaviorForUnmatched,
use ultra_nlp::daachorse::{
segment_fully,
StandardDictionary,
};
let text = " 南京市长江大桥, hello world ";
let dict = StandardDictionary::new(
vec!["南京", "南京市", "市长", "长江", "大桥", "你好世界"]
).unwrap();
let result = segment_fully(text, &dict, BehaviorForUnmatched::KeepAsWords);
assert_eq!(
result
.iter()
.map(|x| x.range().extract(text))
.collect::>(),
vec![
" ", "南京", "南京市", "市长", "长江", "大桥", ", hello world ",
]
);
```