Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Arg name correction: auth_token -> token #1621

Merged
merged 3 commits into from
Oct 24, 2024
Merged

Arg name correction: auth_token -> token #1621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3e2751f
Arg name correction: auth_token -> token
Aug 29, 2024
4617efe
Arg name correction in .rs: auth_token -> token
Sep 3, 2024
15a36c3
update from_pretrained.rs file as well
ArthurZucker Oct 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions bindings/python/py_src/tokenizers/__init__.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -971,7 +971,7 @@ class Tokenizer:
pass

@staticmethod
def from_pretrained(identifier, revision="main", auth_token=None):
def from_pretrained(identifier, revision="main", token=None):
"""
Instantiate a new :class:`~tokenizers.Tokenizer` from an existing file on the
Hugging Face Hub.
Expand All @@ -982,7 +982,7 @@ class Tokenizer:
a tokenizer.json file
revision (:obj:`str`, defaults to `main`):
A branch or commit id
auth_token (:obj:`str`, `optional`, defaults to `None`):
token (:obj:`str`, `optional`, defaults to `None`):
An optional auth token used to access private repositories on the
Hugging Face Hub

Expand Down
12 changes: 6 additions & 6 deletions bindings/python/src/tokenizer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -578,19 +578,19 @@ impl PyTokenizer {
/// a tokenizer.json file
/// revision (:obj:`str`, defaults to `main`):
/// A branch or commit id
/// auth_token (:obj:`str`, `optional`, defaults to `None`):
/// token (:obj:`str`, `optional`, defaults to `None`):
/// An optional auth token used to access private repositories on the
/// Hugging Face Hub
///
/// Returns:
/// :class:`~tokenizers.Tokenizer`: The new tokenizer
#[staticmethod]
#[pyo3(signature = (identifier, revision = String::from("main"), auth_token = None))]
#[pyo3(text_signature = "(identifier, revision=\"main\", auth_token=None)")]
#[pyo3(signature = (identifier, revision = String::from("main"), token = None))]
#[pyo3(text_signature = "(identifier, revision=\"main\", token=None)")]
fn from_pretrained(
identifier: &str,
revision: String,
auth_token: Option<String>,
token: Option<String>,
) -> PyResult<Self> {
let path = Python::with_gil(|py| -> PyResult<String> {
let huggingface_hub = PyModule::import_bound(py, intern!(py, "huggingface_hub"))?;
Expand All @@ -601,8 +601,8 @@ impl PyTokenizer {
(intern!(py, "revision"), &revision),
]
.into_py_dict_bound(py);
if let Some(auth_token) = auth_token {
kwargs.set_item(intern!(py, "token"), auth_token)?;
if let Some(token) = token {
kwargs.set_item(intern!(py, "token"), token)?;
}
let path: String = hf_hub_download.call((), Some(&kwargs))?.extract()?;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we just pass the kwargs to the hf_hub_download api, so IMO this is fine!

Ok(path)
Expand Down
6 changes: 3 additions & 3 deletions tokenizers/src/utils/from_pretrained.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ use std::path::PathBuf;
pub struct FromPretrainedParameters {
pub revision: String,
pub user_agent: HashMap<String, String>,
pub auth_token: Option<String>,
pub token: Option<String>,
}

impl Default for FromPretrainedParameters {
fn default() -> Self {
Self {
revision: "main".into(),
user_agent: HashMap::new(),
auth_token: None,
token: None,
}
}
}
Expand Down Expand Up @@ -60,7 +60,7 @@ pub fn from_pretrained<S: AsRef<str>>(
}

let mut builder = ApiBuilder::new();
if let Some(token) = params.auth_token {
if let Some(token) = params.token {
builder = builder.with_token(Some(token));
}
let api = builder.build()?;
Expand Down
Loading