Skip to content

Commit

Permalink
Create methods to modify Vocabulary in place
Browse files Browse the repository at this point in the history
  • Loading branch information
umut-sahin committed Sep 25, 2024
1 parent 510ee55 commit 2d16373
Showing 1 changed file with 16 additions and 3 deletions.
19 changes: 16 additions & 3 deletions src/vocabulary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ impl Vocabulary {
impl Vocabulary {
/// Inserts a token to the vocabulary with the specified identifier.
pub fn insert(mut self, token: impl Into<Token>, id: TokenId) -> Vocabulary {
let token = token.into();
self.0.entry(token).or_default().push(id);
self.insert_in_place(token, id);
self
}

Expand All @@ -36,11 +35,25 @@ impl Vocabulary {
mut self,
tokens_and_ids: impl IntoIterator<Item = (T, I)>,
) -> Vocabulary {
self.extend_in_place(tokens_and_ids);
self
}
}

impl Vocabulary {
pub fn insert_in_place(&mut self, token: impl Into<Token>, id: TokenId) {
let token = token.into();
self.0.entry(token).or_default().push(id);
}

pub fn extend_in_place<T: Into<Token>, I: IntoIterator<Item = TokenId>>(
&mut self,
tokens_and_ids: impl IntoIterator<Item = (T, I)>,
) {
for (token, ids) in tokens_and_ids.into_iter() {
let token = token.into();
self.0.entry(token).or_default().extend(ids);
}
self
}
}

Expand Down

0 comments on commit 2d16373

Please sign in to comment.