Skip to content

Commit

Permalink
v0.18.1
Browse files Browse the repository at this point in the history
  • Loading branch information
benbrandt committed Oct 25, 2024
1 parent 19f4305 commit 977b0c6
Showing 3 changed files with 10 additions and 4 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Changelog

## v0.18.1

### What's New

- Ensure tokenizer sizers with truncation parameters count their overflow encodings

## v0.18.0

### Breaking
4 changes: 2 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
members = ["bindings/*"]

[workspace.package]
version = "0.18.0"
version = "0.18.1"
authors = ["Ben Brandt <benjamin.j.brandt@gmail.com>"]
edition = "2021"
description = "Split text into semantic chunks, up to a desired chunk size. Supports calculating length by characters and tokens, and is callable from Rust and Python."
@@ -66,7 +66,7 @@ either = "1.6"
itertools = "0.13"
once_cell = "1.20"
pulldown-cmark = { version = "0.12", default-features = false, optional = true }
regex = "1.11.1"
regex = "1.10.6"
rust_tokenizers = { version = "8", optional = true }
strum = { version = "0.26", features = ["derive"] }
thiserror = "1.0.65"

0 comments on commit 977b0c6

Please sign in to comment.