Skip to content
This repository has been archived by the owner on Apr 4, 2023. It is now read-only.

Commit

Permalink
Revert "fix clippy warnings"
Browse files Browse the repository at this point in the history
This reverts commit a1ce3cd.
  • Loading branch information
MarinPostma committed Sep 9, 2021
1 parent a1ce3cd commit 0b767ad
Show file tree
Hide file tree
Showing 29 changed files with 135 additions and 120 deletions.
6 changes: 3 additions & 3 deletions cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ fn indexing_callback(step: milli::update::UpdateIndexingStep, bars: &[ProgressBa
fn documents_from_jsonl(reader: impl Read) -> Result<Vec<u8>> {
let mut writer = Cursor::new(Vec::new());
let mut documents =
milli::documents::DocumentsBuilder::new(&mut writer)?;
milli::documents::DocumentsBuilder::new(&mut writer, bimap::BiHashMap::new())?;

let values = serde_json::Deserializer::from_reader(reader)
.into_iter::<serde_json::Map<String, serde_json::Value>>();
Expand All @@ -216,7 +216,7 @@ fn documents_from_jsonl(reader: impl Read) -> Result<Vec<u8>> {
fn documents_from_json(reader: impl Read) -> Result<Vec<u8>> {
let mut writer = Cursor::new(Vec::new());
let mut documents =
milli::documents::DocumentsBuilder::new(&mut writer)?;
milli::documents::DocumentsBuilder::new(&mut writer, bimap::BiHashMap::new())?;

let json: serde_json::Value = serde_json::from_reader(reader)?;
documents.add_documents(json)?;
Expand All @@ -228,7 +228,7 @@ fn documents_from_json(reader: impl Read) -> Result<Vec<u8>> {
fn documents_from_csv(reader: impl Read) -> Result<Vec<u8>> {
let mut writer = Cursor::new(Vec::new());
let mut documents =
milli::documents::DocumentsBuilder::new(&mut writer)?;
milli::documents::DocumentsBuilder::new(&mut writer, bimap::BiHashMap::new())?;

let mut records = csv::Reader::from_reader(reader);
let iter = records.deserialize::<Map<String, Value>>();
Expand Down
6 changes: 3 additions & 3 deletions http-ui/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1020,7 +1020,7 @@ async fn main() -> anyhow::Result<()> {
fn documents_from_jsonl(reader: impl io::Read) -> anyhow::Result<Vec<u8>> {
let mut writer = Cursor::new(Vec::new());
let mut documents =
milli::documents::DocumentsBuilder::new(&mut writer)?;
milli::documents::DocumentsBuilder::new(&mut writer, bimap::BiHashMap::new())?;

let values = serde_json::Deserializer::from_reader(reader)
.into_iter::<serde_json::Map<String, serde_json::Value>>();
Expand All @@ -1036,7 +1036,7 @@ fn documents_from_jsonl(reader: impl io::Read) -> anyhow::Result<Vec<u8>> {
fn documents_from_json(reader: impl io::Read) -> anyhow::Result<Vec<u8>> {
let mut writer = Cursor::new(Vec::new());
let mut documents =
milli::documents::DocumentsBuilder::new(&mut writer)?;
milli::documents::DocumentsBuilder::new(&mut writer, bimap::BiHashMap::new())?;

let json: serde_json::Value = serde_json::from_reader(reader)?;
documents.add_documents(json)?;
Expand All @@ -1048,7 +1048,7 @@ fn documents_from_json(reader: impl io::Read) -> anyhow::Result<Vec<u8>> {
fn documents_from_csv(reader: impl io::Read) -> anyhow::Result<Vec<u8>> {
let mut writer = Cursor::new(Vec::new());
let mut documents =
milli::documents::DocumentsBuilder::new(&mut writer)?;
milli::documents::DocumentsBuilder::new(&mut writer, bimap::BiHashMap::new())?;

let mut records = csv::Reader::from_reader(reader);
let iter = records.deserialize::<Map<String, Value>>();
Expand Down
7 changes: 4 additions & 3 deletions milli/src/documents/builder.rs
Original file line number Diff line number Diff line change
@@ -1,22 +1,23 @@
use std::io;

use bimap::BiHashMap;
use byteorder::{BigEndian, WriteBytesExt};
use serde::ser::Serialize;

use super::serde::DocumentsSerilializer;
use super::{ByteCounter, DocumentsMetadata, Error, AdditionIndex};
use super::{ByteCounter, DocumentsMetadata, Error};
use crate::FieldId;

pub struct DocumentsBuilder<W> {
serializer: DocumentsSerilializer<W>,
}

impl<W: io::Write + io::Seek> DocumentsBuilder<W> {
pub fn new(writer: W) -> Result<Self, Error> {
pub fn new(writer: W, index: BiHashMap<FieldId, String>) -> Result<Self, Error> {
let mut writer = ByteCounter::new(writer);
// add space to write the offset of the metadata at the end of the writer
writer.write_u64::<BigEndian>(0)?;

let index = AdditionIndex::new();
let serializer =
DocumentsSerilializer { writer, buffer: Vec::new(), index, count: 0, allow_seq: true };

Expand Down
14 changes: 6 additions & 8 deletions milli/src/documents/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,10 @@ pub use reader::DocumentsReader;

use crate::FieldId;

type AdditionIndex = BiHashMap<FieldId, String>;

#[derive(Debug, Serialize, Deserialize)]
struct DocumentsMetadata {
count: usize,
index: AdditionIndex,
index: BiHashMap<FieldId, String>,
}

pub struct ByteCounter<W> {
Expand Down Expand Up @@ -91,7 +89,7 @@ macro_rules! documents {
let documents = serde_json::json!($data);
let mut writer = std::io::Cursor::new(Vec::new());
let mut builder =
crate::documents::DocumentsBuilder::new(&mut writer).unwrap();
crate::documents::DocumentsBuilder::new(&mut writer, bimap::BiHashMap::new()).unwrap();
builder.add_documents(documents).unwrap();
builder.finish().unwrap();

Expand Down Expand Up @@ -122,7 +120,7 @@ mod test {
let mut v = Vec::new();
let mut cursor = io::Cursor::new(&mut v);

let mut builder = DocumentsBuilder::new(&mut cursor).unwrap();
let mut builder = DocumentsBuilder::new(&mut cursor, BiHashMap::new()).unwrap();

builder.add_documents(json).unwrap();

Expand Down Expand Up @@ -151,7 +149,7 @@ mod test {
let mut v = Vec::new();
let mut cursor = io::Cursor::new(&mut v);

let mut builder = DocumentsBuilder::new(&mut cursor).unwrap();
let mut builder = DocumentsBuilder::new(&mut cursor, BiHashMap::new()).unwrap();

builder.add_documents(doc1).unwrap();
builder.add_documents(doc2).unwrap();
Expand Down Expand Up @@ -180,7 +178,7 @@ mod test {
let mut v = Vec::new();
let mut cursor = io::Cursor::new(&mut v);

let mut builder = DocumentsBuilder::new(&mut cursor).unwrap();
let mut builder = DocumentsBuilder::new(&mut cursor, BiHashMap::new()).unwrap();

builder.add_documents(docs).unwrap();

Expand All @@ -203,7 +201,7 @@ mod test {
let mut v = Vec::new();
let mut cursor = io::Cursor::new(&mut v);

let mut builder = DocumentsBuilder::new(&mut cursor).unwrap();
let mut builder = DocumentsBuilder::new(&mut cursor, BiHashMap::new()).unwrap();

let docs = json!([[
{ "toto": false },
Expand Down
11 changes: 6 additions & 5 deletions milli/src/documents/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@ use std::io;
use std::io::{BufReader, Read};
use std::mem::size_of;

use bimap::BiHashMap;
use byteorder::{BigEndian, ReadBytesExt};
use obkv::KvReader;

use super::{DocumentsMetadata, Error, AdditionIndex};
use super::{DocumentsMetadata, Error};
use crate::FieldId;

pub struct DocumentsReader<R> {
Expand Down Expand Up @@ -38,9 +39,9 @@ impl<R: io::Read + io::Seek> DocumentsReader<R> {

/// Returns the next document in the reader, and wraps it in an `obkv::KvReader`, along with a
/// reference to the addition index.
pub fn next_document_with_index(
&mut self,
) -> io::Result<Option<(&AdditionIndex, KvReader<FieldId>)>> {
pub fn next_document_with_index<'a>(
&'a mut self,
) -> io::Result<Option<(&'a BiHashMap<FieldId, String>, KvReader<'a, FieldId>)>> {
if self.seen_documents < self.metadata.count {
let doc_len = self.reader.read_u32::<BigEndian>()?;
self.buffer.resize(doc_len as usize, 0);
Expand All @@ -55,7 +56,7 @@ impl<R: io::Read + io::Seek> DocumentsReader<R> {
}

/// Return the fields index for the documents batch.
pub fn index(&self) -> &AdditionIndex {
pub fn index(&self) -> &BiHashMap<FieldId, String> {
&self.metadata.index
}

Expand Down
11 changes: 6 additions & 5 deletions milli/src/documents/serde.rs
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
use std::convert::TryInto;
use std::{fmt, io};

use bimap::BiHashMap;
use byteorder::{BigEndian, WriteBytesExt};
use obkv::KvWriter;
use serde::ser::{Impossible, Serialize, SerializeMap, SerializeSeq, Serializer};

use super::{AdditionIndex, ByteCounter, Error};
use super::{ByteCounter, Error};
use crate::FieldId;

pub struct DocumentsSerilializer<W> {
pub writer: ByteCounter<W>,
pub buffer: Vec<u8>,
pub index: AdditionIndex,
pub index: BiHashMap<FieldId, String>,
pub count: usize,
pub allow_seq: bool,
}
Expand Down Expand Up @@ -224,7 +225,7 @@ impl<'a, W: io::Write> SerializeSeq for SeqSerializer<'a, W> {

pub struct MapSerializer<'a, W> {
map: KvWriter<io::Cursor<&'a mut Vec<u8>>, FieldId>,
index: &'a mut AdditionIndex,
index: &'a mut BiHashMap<FieldId, String>,
writer: W,
buffer: Vec<u8>,
}
Expand All @@ -248,7 +249,7 @@ impl<'a, W: io::Write> SerializeMap for MapSerializer<'a, W> {
let data_len: u32 = data.len().try_into().map_err(|_| Error::DocumentTooLarge)?;

self.writer.write_u32::<BigEndian>(data_len).map_err(Error::Io)?;
self.writer.write_all(data).map_err(Error::Io)?;
self.writer.write_all(&data).map_err(Error::Io)?;

Ok(())
}
Expand Down Expand Up @@ -276,7 +277,7 @@ impl<'a, W: io::Write> SerializeMap for MapSerializer<'a, W> {
}

struct FieldSerializer<'a> {
index: &'a mut AdditionIndex,
index: &'a mut BiHashMap<FieldId, String>,
}

impl<'a> serde::Serializer for FieldSerializer<'a> {
Expand Down
2 changes: 1 addition & 1 deletion milli/src/fields_ids_map.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ impl FieldsIdsMap {
}

/// Iterate over the ids in the order of the ids.
pub fn ids(&self) -> impl Iterator<Item = FieldId> + '_ {
pub fn ids<'a>(&'a self) -> impl Iterator<Item = FieldId> + 'a {
self.ids_names.keys().copied()
}

Expand Down
12 changes: 8 additions & 4 deletions milli/src/heed_codec/facet/facet_level_value_f64_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ impl heed::BytesEncode<'_> for FacetLevelValueF64Codec {
fn bytes_encode((field_id, level, left, right): &Self::EItem) -> Option<Cow<[u8]>> {
let mut buffer = [0u8; 32];

// Write the globally ordered floats.
let bytes = f64_into_bytes(*left)?;
buffer[..8].copy_from_slice(&bytes[..]);

let len = if *level != 0 {
// Write the globally ordered floats.
let bytes = f64_into_bytes(*left)?;
buffer[..8].copy_from_slice(&bytes[..]);

let bytes = f64_into_bytes(*right)?;
buffer[8..16].copy_from_slice(&bytes[..]);

Expand All @@ -51,6 +51,10 @@ impl heed::BytesEncode<'_> for FacetLevelValueF64Codec {

32 // length
} else {
// Write the globally ordered floats.
let bytes = f64_into_bytes(*left)?;
buffer[..8].copy_from_slice(&bytes[..]);

// Then the f64 values just to be able to read them back.
let bytes = left.to_be_bytes();
buffer[8..16].copy_from_slice(&bytes[..]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ where
type EItem = (&'a str, C::EItem);

fn bytes_encode((string, value): &'a Self::EItem) -> Option<Cow<[u8]>> {
let value_bytes = C::bytes_encode(value)?;
let value_bytes = C::bytes_encode(&value)?;

let mut bytes = Vec::with_capacity(2 + string.len() + value_bytes.len());
encode_prefix_string(string, &mut bytes).ok()?;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,14 @@ where
bytes.extend_from_slice(left.as_bytes());
bytes.extend_from_slice(right.as_bytes());

let value_bytes = C::bytes_encode(value)?;
let value_bytes = C::bytes_encode(&value)?;
bytes.extend_from_slice(&value_bytes[..]);

Some(Cow::Owned(bytes))
}
None => {
bytes.push(0);
let value_bytes = C::bytes_encode(value)?;
let value_bytes = C::bytes_encode(&value)?;
bytes.extend_from_slice(&value_bytes[..]);
Some(Cow::Owned(bytes))
}
Expand Down
6 changes: 3 additions & 3 deletions milli/src/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ impl Index {
/// Writes the documents primary key, this is the field name that is used to store the id.
pub(crate) fn put_primary_key(&self, wtxn: &mut RwTxn, primary_key: &str) -> heed::Result<()> {
self.set_updated_at(wtxn, &Utc::now())?;
self.main.put::<_, Str, Str>(wtxn, main_key::PRIMARY_KEY_KEY, primary_key)
self.main.put::<_, Str, Str>(wtxn, main_key::PRIMARY_KEY_KEY, &primary_key)
}

/// Deletes the primary key of the documents, this can be done to reset indexes settings.
Expand Down Expand Up @@ -741,7 +741,7 @@ impl Index {
let kv = self
.documents
.get(rtxn, &BEU32::new(id))?
.ok_or(UserError::UnknownInternalDocumentId { document_id: id })?;
.ok_or_else(|| UserError::UnknownInternalDocumentId { document_id: id })?;
documents.push((id, kv));
}

Expand Down Expand Up @@ -795,7 +795,7 @@ impl Index {
wtxn: &mut RwTxn,
time: &DateTime<Utc>,
) -> heed::Result<()> {
self.main.put::<_, Str, SerdeJson<DateTime<Utc>>>(wtxn, main_key::UPDATED_AT_KEY, time)
self.main.put::<_, Str, SerdeJson<DateTime<Utc>>>(wtxn, main_key::UPDATED_AT_KEY, &time)
}
}

Expand Down
2 changes: 1 addition & 1 deletion milli/src/search/criteria/asc_desc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ impl<'t> Criterion for AscDesc<'t> {
let mut candidates = match (&self.query_tree, candidates) {
(_, Some(candidates)) => candidates,
(Some(qt), None) => {
let context = CriteriaBuilder::new(self.rtxn, self.index)?;
let context = CriteriaBuilder::new(&self.rtxn, &self.index)?;
resolve_query_tree(&context, qt, params.wdcache)?
}
(None, None) => self.index.documents_ids(self.rtxn)?,
Expand Down
6 changes: 3 additions & 3 deletions milli/src/search/criteria/attribute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -288,15 +288,15 @@ impl<'t, 'q> QueryLevelIterator<'t, 'q> {
for query in queries {
match &query.kind {
QueryKind::Exact { word, .. } => {
if !query.prefix || ctx.in_prefix_cache(word) {
if !query.prefix || ctx.in_prefix_cache(&word) {
let word = Cow::Borrowed(query.kind.word());
if let Some(word_level_iterator) =
WordLevelIterator::new(ctx, word, query.prefix)?
{
inner.push(word_level_iterator);
}
} else {
for (word, _) in word_derivations(word, true, 0, ctx.words_fst(), wdcache)?
for (word, _) in word_derivations(&word, true, 0, ctx.words_fst(), wdcache)?
{
let word = Cow::Owned(word.to_owned());
if let Some(word_level_iterator) =
Expand All @@ -309,7 +309,7 @@ impl<'t, 'q> QueryLevelIterator<'t, 'q> {
}
QueryKind::Tolerant { typo, word } => {
for (word, _) in
word_derivations(word, query.prefix, *typo, ctx.words_fst(), wdcache)?
word_derivations(&word, query.prefix, *typo, ctx.words_fst(), wdcache)?
{
let word = Cow::Owned(word.to_owned());
if let Some(word_level_iterator) = WordLevelIterator::new(ctx, word, false)?
Expand Down
Loading

0 comments on commit 0b767ad

Please sign in to comment.