diff --git a/docs/sorting.md b/docs/sorting.md index 5ca9a8301..dfe5681e4 100644 --- a/docs/sorting.md +++ b/docs/sorting.md @@ -96,6 +96,16 @@ With the combination of `ISearchResult.Skip` and `maxResults`, we can tell Lucen * Skip over a certain number of results without allocating them and tell Lucene * only allocate a certain number of results after skipping +### Deep Paging +When using Lucene.NET as the Examine provider it is possible to more efficiently perform deep paging. +Steps: +1. Build and execute your query as normal. +2. Cast the ISearchResults from IQueryExecutor.Execute to ILuceneSearchResults +3. Store ILuceneSearchResults.SearchAfter (SearchAfterOptions) for the next page. +4. Create the same query as the previous request. +5. When calling IQueryExecutor.Execute. Pass in new LuceneQueryOptions(skip,take, SearchAfterOptions); Skip will be ignored, the next take documents will be retrieved after the SearchAfterOptions document. +6. Repeat Steps 2-5 for each page. + ### Example ```cs diff --git a/docs/v2/articles/configuration.md b/docs/v2/articles/configuration.md index 330d9551e..59103ad68 100644 --- a/docs/v2/articles/configuration.md +++ b/docs/v2/articles/configuration.md @@ -98,24 +98,47 @@ Value types are responsible for: These are the default field value types provided with Examine. Each value type can be resolved from the static class [`Examine.FieldDefinitionTypes`](xref:Examine.FieldDefinitionTypes) (i.e. [`Examine.FieldDefinitionTypes.FullText`](xref:Examine.FieldDefinitionTypes#Examine_FieldDefinitionTypes_FullText)). -| Value Type | Description | Sortable | -|----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------| -| FullText | __Default__.
The field will be indexed with the index's
default Analyzer without any sortability. 
Generally this is fine for normal text searching. | ❌ | -| FullTextSortable | Will be indexed with FullText but also 
enable sorting on this field for search results. 
_FullText sortability adds additional overhead 
since it requires an additional index field._ | ✅ | -| Integer | Stored as a numerical structure. | ✅ | -| Float | Stored as a numerical structure. | ✅ | -| Double | Stored as a numerical structure. | ✅ | -| Long | Stored as a numerical structure. | ✅ | -| DateTime | Stored as a DateTime, 
represented by a numerical structure. | ✅ | -| DateYear | Just like DateTime but with 
precision only to the year. | ✅ | -| DateMonth | Just like DateTime but with 
precision only to the month. | ✅ | -| DateDay | Just like DateTime but with 
precision only to the day. | ✅ | -| DateHour | Just like DateTime but with 
precision only to the hour. | ✅ | -| DateMinute | Just like DateTime but with 
precision only to the minute. | ✅ | -| EmailAddress | Uses custom analyzers for dealing 
with email address searching. | ❌ | -| InvariantCultureIgnoreCase | Uses custom analyzers for dealing with text so it
 can be searched on regardless of the culture/casing. | ❌ | -| Raw | Will be indexed without analysis, searching will
 only match with an exact value. | ❌ | - +| Value Type | Description | Sortable | Facetable | Retrievable | Searchable | Filterable | Analyzer | +| ------------------------------ | ------------ | -------- | --------- | ----------- | ---------- | ---------- | -------- | +| FullText | **Default**. The field will be indexed with the index's default Analyzer without any sortability. Generally this is fine for normal text searching. | ❌ | ❌ | ✅ | ✅ | ✅ | CultureInvariantStandardAnalyzer or Index default | +| FullTextSortable | Will be indexed with FullText but also enable sorting on this field for search results. *FullText sortability adds additional overhead since it requires an additional index field.* | ✅ | ❌ | ✅ | ✅ | ✅ | CultureInvariantStandardAnalyzer or Index default | +| Integer | Stored as a numerical structure.| ✅ | ❌ | ✅ | ❌ | ✅ | - | +| Float | Stored as a numerical structure. | ✅ | ❌ | ✅ | ❌ | ✅ | - | +| Double | Stored as a numerical structure. | ✅ | ❌ | ✅ | ❌ | ✅ | - | +| Long | Stored as a numerical structure. | ✅ | ❌ | ✅ | ❌ | ✅ | - | +| DateTime | Stored as a DateTime, represented by a numerical structure. | ✅ | ❌ | ✅ | ❌ | ✅ | - | +| DateYear | Just like DateTime but with precision only to the year. | ✅ | ❌ | ✅ | ❌ | ✅ | - | +| DateMonth | Just like DateTime but with precision only to the month. | ✅ | ❌ | ✅ | ❌ | ✅ | - | +| DateDay | Just like DateTime but with precision only to the day. | ✅ | ❌ | ✅ | ❌ | ✅ | - | +| DateHour | Just like DateTime but with precision only to the hour. | ✅ | ❌ | ✅ | ❌ | ✅ | - | +| DateMinute | Just like DateTime but with precision only to the minute. | ✅ | ❌ | ✅ | ❌ | ✅ | - | +| EmailAddress | Uses custom analyzers for dealing with email address searching. | ❌ | ❌ | ✅ | ✅ | ✅ | EmailAddressAnalyzer | +| InvariantCultureIgnoreCase | Uses custom analyzers for dealing with text so it can be searched on regardless of the culture/casing. | ❌ | ❌ | ✅ | ✅ | ✅ | CultureInvariantStandardAnalyzer | +| Raw | Will be indexed without analysis, searching will only match with an exact value. | ❌ | ❌ | ✅ | ✅ | ✅ | KeywordAnalyzer | +| FacetFullText | The field will be indexed with the index's default Analyzer without any sortability. Generally this is fine for normal text searching. | ❌ | ✅ | ✅ | ✅ | ✅ | CultureInvariantStandardAnalyzer or Index default | +| FacetFullTextSortable | Will be indexed with FullText but also enable sorting on this field for search results. *FullText sortability adds additional overhead since it requires an additional index field.* | ✅ | ✅ | ✅ | ✅ | ✅ | CultureInvariantStandardAnalyzer or Index default | +| FacetInteger | Stored as a numerical structure. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetFloat | Stored as a numerical structure. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetDouble | Stored as a numerical structure. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetLong | Stored as a numerical structure. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetDateTime | Stored as a DateTime, represented by a numerical structure. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetDateYear | Just like DateTime but with precision only to the year. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetDateMonth | Just like DateTime but with precision only to the month. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetDateDay | Just like DateTime but with precision only to the day. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetDateHour | Just like DateTime but with precision only to the hour. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetDateMinute | Just like DateTime but with precision only to the minute. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetTaxonomyFullText | The field will be indexed with the index's default Analyzer without any sortability. Generally this is fine for normal text searching. Stored in the Taxonomy Facet sidecar index. | ❌ | ✅ | ✅ | ✅ | ✅ | CultureInvariantStandardAnalyzer or Index default | +| FacetTaxonomyFullTextSortable | Will be indexed with FullText but also enable sorting on this field for search results. *FullText sortability adds additional overhead since it requires an additional index field.* Stored in the Taxonomy Facet sidecar index. | ✅ | ✅ | ✅ | ✅ | ✅ | CultureInvariantStandardAnalyzer or Index default | +| FacetTaxonomyInteger | Stored as a numerical structure. Stored in the Taxonomy Facet sidecar index. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetTaxonomyFloat | Stored as a numerical structure. Stored in the Taxonomy Facet sidecar index. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetTaxonomyDouble | Stored as a numerical structure. Stored in the Taxonomy Facet sidecar index. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetTaxonomyLong | Stored as a numerical structure. Stored in the Taxonomy Facet sidecar index. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetTaxonomyDateTime | Stored as a DateTime, represented by a numerical structure. Stored in the Taxonomy Facet sidecar index. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetTaxonomyDateYear | Just like DateTime but with precision only to the year. Stored in the Taxonomy Facet sidecar index. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetTaxonomyDateMonth | Just like DateTime but with precision only to the month. Stored in the Taxonomy Facet sidecar index. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetTaxonomyDateDay | Just like DateTime but with precision only to the day. Stored in the Taxonomy Facet sidecar index. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetTaxonomyDateHour | Just like DateTime but with precision only to the hour. Stored in the Taxonomy Facet sidecar index. | ✅ |✅ | ✅ | ❌ | ✅ | - | +| FacetTaxonomyDateMinute | Just like DateTime but with precision only to the minute. Stored in the Taxonomy Facet sidecar index. | ✅ |✅ | ✅ | ❌ | ✅ | - | ### Custom field value types A field value type is defined by [`IIndexFieldValueType`](xref:Examine.Lucene.Indexing.IIndexFieldValueType) @@ -192,3 +215,70 @@ That returns an result [`ValueSetValidationResult`](xref:Examine.ValueSetValidat * `Filtered` - The ValueSet has been filtered/modified by the validator and will be indexed Examine only has one implementation: [`ValueSetValidatorDelegate`](xref:Examine.Lucene.Providers.ValueSetValidatorDelegate) which can be used by developers as a simple way to create a validator based on a callback, else developers can implement this interface if required. By default, no ValueSet validation is done with Examine. + +## Facets configuration + +When using the facets feature it's possible to add facets configuration to change the behavior of the indexing. + +For example, you can allow multiple values in an indexed field with the configuration below. +```csharp +// Create a config +var facetsConfig = new FacetsConfig(); + +// Set field to be able to contain multiple values (This is default for a field in Examine. But you only need this if you are actually using multiple values for a single field) +facetsConfig.SetMultiValued("MultiIdField", true); + +services.AddExamineLuceneIndex("MyIndex", + // Set the indexing of your fields to use the facet type + fieldDefinitions: new FieldDefinitionCollection( + new FieldDefinition("Timestamp", FieldDefinitionTypes.FacetDateTime), + + new FieldDefinition("MultiIdField", FieldDefinitionTypes.FacetFullText) + ), + // Pass your config + facetsConfig: facetsConfig + ); +``` + +Without this configuration for multiple values, you'll notice that your faceted search breaks or behaves differently than expected. + +### Hierarchical and Taxonomy Facets configuration + +To enable support for hierarchical facets as well as supporting faster faceting the Taxonomy Facet sidecar index can be enabled. + +1. Set LuceneIndexOptions.UseTaxonomyIndex = true; for the index. This enables the use of the Taxonomy sidecar index. +2. Change the Field Definitions to use the "FacetTaxonomy" Field Definition Types instead of the "Facet" types. E.g. FieldDefinitionTypes.FacetFullText => FieldDefinitionTypes.FacetTaxonomyFullText. +3. To enable hierarchical facets on a field, call FacetsConfig.SetHierarchical("facetfieldname", true); + +Example: + +```csharp +// Create a config +var facetsConfig = new FacetsConfig(); + +// Set field to be able to support hierarchical facets +facetsConfig.SetHierarchical("hierarchyFacetfield", true); + +// Set field to be able to contain multiple values (This is default for a field in Examine. But you only need this if you are actually using multiple values for a single field) +facetsConfig.SetMultiValued("MultiIdField", true); + +services.AddExamineLuceneIndex("MyIndex", + // Set the indexing of your fields to use the facet Taxonomy type + fieldDefinitions: new FieldDefinitionCollection( + new FieldDefinition("Timestamp", FieldDefinitionTypes.FacetTaxonomyDateTime), + new FieldDefinition("hierarchyFacetfield", FieldDefinitionTypes.FacetTaxonomyFullText), + + new FieldDefinition("MultiIdField", FieldDefinitionTypes.FacetTaxonomyFullText) + ), + // Pass your config + facetsConfig: facetsConfig, + // Enable the Taxonomy sidecar index + useTaxonomyIndex: true + ); +``` + +**Note: See more examples of how facets configuration can be used under [Searching](xref:searching)** + +To explore other configuration settings see the links below: +- [FacetsConfig API docs](https://lucenenet.apache.org/docs/4.8.0-beta00016/api/facet/Lucene.Net.Facet.FacetsConfig.html#methods) +- [Facets with lucene](https://norconex.com/facets-with-lucene/). See how the config is used in the code examples. \ No newline at end of file diff --git a/docs/v2/articles/searching.md b/docs/v2/articles/searching.md index 017fc1e68..18dd2d5c1 100644 --- a/docs/v2/articles/searching.md +++ b/docs/v2/articles/searching.md @@ -226,6 +226,218 @@ var query = searcher.CreateQuery() This will match for example: `test`, `tests` and `tester` + +## Faceting + +### String Facets + +String facets allows for counting the documents that share the same string value. This type of faceting is possible on all faceted index type. + +#### Basic example + +```csharp +var searcher = myIndex.Searcher; +var results = searcher.CreateQuery() + .Field("Address", "Hills") + .WithFacets(facets => facets.Facet("Address")) // Get facets of the Address field + .Execute(); + +var addressFacetResults = results.GetFacet("Address"); // Returns the facets for the specific field Address + +/* +* Example value +* Label: Hills, Value: 2 +* Label: Hollywood, Value: 10 +*/ + +var hillsValue = addressFacetResults.Facet("Hills"); // Gets the IFacetValue for the facet Hills +``` + +#### Filtered value example + +```csharp +var searcher = myIndex.Searcher; +var results = searcher.CreateQuery() + .Field("Address", "Hills") + .WithFacets(facets => facets.Facet("Address", "Hills")) // Get facets of the Address field + .Execute(); + +var addressFacetResults = results.GetFacet("Address"); // Returns the facets for the specific field Address + +/* +* Example value +* Label: Hills, Value: 2 <-- As Hills was the only filtered value we will only get this facet +*/ + +var hillsValue = addressFacetResults.Facet("Hills"); // Gets the IFacetValue for the facet Hills +``` + +#### MaxCount example + +```csharp +var searcher = myIndex.Searcher; +var results = searcher.CreateQuery() + .Field("Address", "Hills") + .WithFacets(facets => facets.Facet("Address")) // Get facets of the Address field + .Execute(); + +var addressFacetResults = results.GetFacet("Address"); // Returns the facets for the specific field Address + +/* +* Example value +* Label: Hills, Value: 2 +* Label: Hollywood, Value: 10 +* Label: London, Value: 12 +*/ + +results = searcher.CreateQuery() + .Field("Address", "Hills") + .WithFacets(facets => facets.Facet("Address", config => config.MaxCount(2))) // Get facets of the Address field & Gets the top 2 results (The facets with the highest value) + .Execute(); + +addressFacetResults = results.GetFacet("Address"); // Returns the facets for the specific field Address + +/* +* Example value (Notice only 2 values are present) +* Label: Hollywood, Value: 10 +* Label: London, Value: 12 +*/ +``` + +#### FacetField example + +```csharp +// Setup + +// Create a config +var facetsConfig = new FacetsConfig(); + +// Set the index field name to facet_address. This will store facets of this field under facet_address instead of the default $facets. This requires you to use FacetField in your Facet query. (Only works on string facets). +facetsConfig.SetIndexFieldName("Address", "facet_address"); + +services.AddExamineLuceneIndex("MyIndex", + // Set the indexing of your fields to use the facet type + fieldDefinitions: new FieldDefinitionCollection( + new FieldDefinition("Address", FieldDefinitionTypes.FacetFullText) + ), + // Pass your config + facetsConfig: facetsConfig + ); + + +var searcher = myIndex.Searcher; +var results = searcher.CreateQuery() + .Field("Address", "Hills") + .WithFacets(facets => facets.Facet("Address")) // Get facets of the Address field from the facet field address_facet (The facet field is automatically read from the FacetsConfig) + .Execute(); + +var addressFacetResults = results.GetFacet("Address"); // Returns the facets for the specific field Address + +/* +* Example value +* Label: Hills, Value: 2 +* Label: Hollywood, Value: 10 +*/ +``` + +### Numeric Range facet + +Numeric range facets can be used with numbers and get facets for numeric ranges. For example, it would group documents of the same price range. + +There's two categories of numeric ranges - `DoubleRanges` and `Int64Range` for double/float values and int/long/datetime values respectively. + +#### Double range example + +```csharp +var searcher = myIndex.Searcher; +var results = searcher.CreateQuery() + .All() + .WithFacets(facets => facets.Facet("Price", new DoubleRange[] { + new DoubleRange("0-10", 0, true, 10, true), + new DoubleRange("11-20", 11, true, 20, true) + })) // Get facets of the price field + .Execute(); + +var priceFacetResults = results.GetFacet("Price"); // Returns the facets for the specific field Price + +/* +* Example value +* Label: 0-10, Value: 2 +* Label: 11-20, Value: 10 +*/ + +var firstRangeValue = priceFacetResults.Facet("0-10"); // Gets the IFacetValue for the facet "0-10" +``` + +#### Float range example + +```csharp +var searcher = myIndex.Searcher; +var results = searcher.CreateQuery() + .All() + .WithFacets(facets => facets.Facet("Price", new FloatRange[] { + new FloatRange("0-10", 0, true, 10, true), + new FloatRange("11-20", 11, true, 20, true) + })) // Get facets of the price field + .Execute(); + +var priceFacetResults = results.GetFacet("Price"); // Returns the facets for the specific field Price + +/* +* Example value +* Label: 0-10, Value: 2 +* Label: 11-20, Value: 10 +*/ + +var firstRangeValue = priceFacetResults.Facet("0-10"); // Gets the IFacetValue for the facet "0-10" +``` + +#### Int/Long range example + +```csharp +var searcher = myIndex.Searcher; +var results = searcher.CreateQuery() + .All() + .WithFacets(facets => facets.Facet("Price", new Int64Range[] { + new Int64Range("0-10", 0, true, 10, true), + new Int64Range("11-20", 11, true, 20, true) + })) // Get facets of the price field + .Execute(); + +var priceFacetResults = results.GetFacet("Price"); // Returns the facets for the specific field Price + +/* +* Example value +* Label: 0-10, Value: 2 +* Label: 11-20, Value: 10 +*/ + +var firstRangeValue = priceFacetResults.Facet("0-10"); // Gets the IFacetValue for the facet "0-10" +``` + +#### DateTime range example + +```csharp +var searcher = myIndex.Searcher; +var results = searcher.CreateQuery() + .All() + .WithFacets(facets => facets.Facet("Created", new Int64Range[] { + new Int64Range("first", DateTime.UtcNow.AddDays(-1).Ticks, true, DateTime.UtcNow.Ticks, true), + new Int64Range("last", DateTime.UtcNow.AddDays(1).Ticks, true, DateTime.UtcNow.AddDays(2).Ticks, true) + })) // Get facets of the price field + .Execute(); + +var createdFacetResults = results.GetFacet("Created"); // Returns the facets for the specific field Created + +/* +* Example value +* Label: first, Value: 2 +* Label: last, Value: 10 +*/ + +var firstRangeValue = createdFacetResults.Facet("first"); // Gets the IFacetValue for the facet "first" +``` + ## Lucene queries Find a reference to how to write Lucene queries in the [Lucene 4.8.0 docs](https://lucene.apache.org/core/4_8_0/queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package_description). @@ -257,4 +469,4 @@ var query = searcher.CreateQuery(); var query = (LuceneSearchQuery)query.NativeQuery("hello:world").And(); // Make query ready for extending query.LuceneQuery(NumericRangeQuery.NewInt64Range("numTest", 4, 5, true, true)); // Add the raw lucene query var results = query.Execute(); -``` \ No newline at end of file +``` diff --git a/docs/v2/articles/sorting.md b/docs/v2/articles/sorting.md index 3b5a8565a..66d062ee1 100644 --- a/docs/v2/articles/sorting.md +++ b/docs/v2/articles/sorting.md @@ -77,3 +77,15 @@ var takeSevenHundredResults = searcher ``` By default when using [`Execute()`](xref:Examine.Search.IQueryExecutor#Examine_Search_IQueryExecutor_Execute_Examine_Search_QueryOptions_) or `Execute(QueryOptions.SkipTake(0))` where no take parameter is provided the take of the search will be set to [`QueryOptions.DefaultMaxResults`](xref:Examine.Search.QueryOptions#Examine_Search_QueryOptions_DefaultMaxResults) (500). + +## Deep Paging + +When using Lucene.NET as the Examine provider it is possible to more efficiently perform deep paging. +Steps: + +1. Build and execute your query as normal. +2. Cast the ISearchResults from IQueryExecutor.Execute to ILuceneSearchResults +3. Store ILuceneSearchResults.SearchAfter (SearchAfterOptions) for the next page. +4. Create the same query as the previous request. +5. When calling IQueryExecutor.Execute. Pass in new LuceneQueryOptions(skip,take, SearchAfterOptions); Skip will be ignored, the next take documents will be retrieved after the SearchAfterOptions document. +6. Repeat Steps 2-5 for each page. \ No newline at end of file diff --git a/src/Examine.Core/FieldDefinitionTypes.cs b/src/Examine.Core/FieldDefinitionTypes.cs index 619c41909..5fe13cb35 100644 --- a/src/Examine.Core/FieldDefinitionTypes.cs +++ b/src/Examine.Core/FieldDefinitionTypes.cs @@ -141,5 +141,67 @@ public static class FieldDefinitionTypes /// Facetable version of /// public const string FacetFullTextSortable = "facetfulltextsortable"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyInteger = "facettaxonomyint"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyFloat = "facettaxonomyfloat"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyDouble = "facettaxonomydouble"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyLong = "facettaxonomylong"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyDateTime = "facettaxonomydatetime"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyDateYear = "facettaxonomydate.year"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyDateMonth = "facettaxonomydate.month"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyDateDay = "facettaxonomydate.day"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyDateHour = "facettaxonomydate.hour"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyDateMinute = "facettaxonomydate.minute"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyFullText = "facettaxonomyfulltext"; + + /// + /// Facetable version of stored in the Taxonomy Index + /// + public const string FacetTaxonomyFullTextSortable = "facettaxonomyfulltextsortable"; + + } } diff --git a/src/Examine.Core/Search/FacetLabel.cs b/src/Examine.Core/Search/FacetLabel.cs new file mode 100644 index 000000000..1723ccc3b --- /dev/null +++ b/src/Examine.Core/Search/FacetLabel.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; + +namespace Examine.Search +{ + /// + /// Holds a sequence of string components, specifying the hierarchical name of a category. + /// + public readonly struct FacetLabel : IFacetLabel + { + private readonly string[] _components; + + /// + /// Constructor + /// + /// The components of this FacetLabel + public FacetLabel(string[] components) + { + _components = components; + } + + /// + /// Constructor + /// + /// The name of the dimension that stores this FacetLabel + /// >The components of this FacetLabel + public FacetLabel(string dimension, string[] components) + { + _components = new string[1 + components.Length]; + _components[0] = dimension; + Array.Copy(components, 0, _components, 1, components.Length); + } + + /// + public string[] Components => _components; + + /// + public int Length => _components.Length; + + // From Lucene.NET + public int CompareTo(IFacetLabel other) + { + int len = Length < other.Length ? Length : other.Length; + for (int i = 0, j = 0; i < len; i++, j++) + { + int cmp = StringComparer.Ordinal.Compare(Components[i], other.Components[j]); + if (cmp < 0) + { + return -1; // this is 'before' + } + if (cmp > 0) + { + return 1; // this is 'after' + } + } + + // one is a prefix of the other + return Length - other.Length; + } + + + /// + public IFacetLabel Subpath(int length) + { + if(Components.Length <= length) + { + return new FacetLabel(Components); + } + + List subpathComponents = new List(length); + int index = 0; + while (index < length && index < _components.Length) + { + subpathComponents.Add(_components[index]); + index++; + } + return new FacetLabel(subpathComponents.ToArray()); + } + } +} diff --git a/src/Examine.Core/Search/IFacetLabel.cs b/src/Examine.Core/Search/IFacetLabel.cs new file mode 100644 index 000000000..d9134aee2 --- /dev/null +++ b/src/Examine.Core/Search/IFacetLabel.cs @@ -0,0 +1,25 @@ +using System; + +namespace Examine.Search +{ + /// + /// Holds a sequence of string components, specifying the hierarchical name of a category. + /// + public interface IFacetLabel : IComparable + { + /// + /// The components of this IFacetLabel. + /// + string[] Components { get; } + + /// + /// The number of components of this IFacetLabel. + /// + int Length { get; } + + /// + /// Returns a sub-path of this path up to length components. + /// + IFacetLabel Subpath(int length); + } +} diff --git a/src/Examine.Core/Search/IFacetQueryField.cs b/src/Examine.Core/Search/IFacetQueryField.cs index 1d6d4521b..26fd0ffc7 100644 --- a/src/Examine.Core/Search/IFacetQueryField.cs +++ b/src/Examine.Core/Search/IFacetQueryField.cs @@ -11,8 +11,9 @@ public interface IFacetQueryField IFacetQueryField MaxCount(int count); /// - /// Path Hierarchy + /// Set the Facet Path /// - IFacetQueryField SetPath(string[] path); + /// Facet Path + IFacetQueryField SetPath(params string[] path); } } diff --git a/src/Examine.Core/Search/QueryOptions.cs b/src/Examine.Core/Search/QueryOptions.cs index c3974cc67..587468a0b 100644 --- a/src/Examine.Core/Search/QueryOptions.cs +++ b/src/Examine.Core/Search/QueryOptions.cs @@ -43,12 +43,12 @@ public QueryOptions(int skip, int? take = null) } /// - /// The ammount of items to skip + /// The number of documents to skip in the result set. /// public int Skip { get; } /// - /// The ammount of items to take + /// The number of documents to take in the result set. /// public int Take { get; } } diff --git a/src/Examine.Host/ServicesCollectionExtensions.cs b/src/Examine.Host/ServicesCollectionExtensions.cs index 6e1cb6e27..8594e121d 100644 --- a/src/Examine.Host/ServicesCollectionExtensions.cs +++ b/src/Examine.Host/ServicesCollectionExtensions.cs @@ -19,6 +19,7 @@ namespace Examine /// public static class ServicesCollectionExtensions { + /// /// Registers a file system based Lucene Examine index /// @@ -88,6 +89,50 @@ IOptionsMonitor options }); } + /// + /// Registers an Examine index + /// + public static IServiceCollection AddExamineLuceneIndex( + this IServiceCollection serviceCollection, + string name, + FieldDefinitionCollection? fieldDefinitions = null, + Analyzer? analyzer = null, + IValueSetValidator? validator = null, + IReadOnlyDictionary? indexValueTypesFactory = null, + FacetsConfig? facetsConfig = null, + bool useTaxonomyIndex = false) + where TIndex : LuceneIndex + where TDirectoryFactory : class, IDirectoryFactory + { + // This is the long way to add IOptions but gives us access to the + // services collection which we need to get the dir factory + serviceCollection.AddSingleton>( + services => new ConfigureNamedOptions( + name, + (options) => + { + options.Analyzer = analyzer; + options.Validator = validator; + options.IndexValueTypesFactory = indexValueTypesFactory; + options.FieldDefinitions = fieldDefinitions ?? options.FieldDefinitions; + options.DirectoryFactory = services.GetRequiredService(); + options.FacetsConfig = facetsConfig ?? new FacetsConfig(); + options.UseTaxonomyIndex = useTaxonomyIndex; + })); + + return serviceCollection.AddSingleton(services => + { + IOptionsMonitor options + = services.GetRequiredService>(); + + TIndex index = ActivatorUtilities.CreateInstance( + services, + new object[] { name, options }); + + return index; + }); + } + /// /// Registers a standalone Examine searcher /// @@ -122,7 +167,8 @@ public static IServiceCollection AddExamineLuceneMultiSearcher( this IServiceCollection serviceCollection, string name, string[] indexNames, - Analyzer? analyzer = null) + Analyzer analyzer = null, + FacetsConfig facetsConfig = null) => serviceCollection.AddExamineSearcher(name, s => { IEnumerable matchedIndexes = s.GetServices() @@ -130,9 +176,14 @@ public static IServiceCollection AddExamineLuceneMultiSearcher( var parameters = new List { - matchedIndexes + matchedIndexes, }; + if (facetsConfig != null) + { + parameters.Add(facetsConfig); + } + if (analyzer != null) { parameters.Add(analyzer); diff --git a/src/Examine.Lucene/Directories/DirectoryFactory.cs b/src/Examine.Lucene/Directories/DirectoryFactory.cs index 64dc1158e..c55000baf 100644 --- a/src/Examine.Lucene/Directories/DirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/DirectoryFactory.cs @@ -8,9 +8,14 @@ namespace Examine.Lucene.Directories public class GenericDirectoryFactory : DirectoryFactoryBase { private readonly Func _factory; - + private readonly Func _taxonomyDirectoryFactory; + /// - public GenericDirectoryFactory(Func factory) => _factory = factory; + public GenericDirectoryFactory(Func factory, Func taxonomyDirectoryFactory = null) + { + _factory = factory; + _taxonomyDirectoryFactory = taxonomyDirectoryFactory; + } /// protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) @@ -22,5 +27,16 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force } return dir; } + + /// + protected override Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock) + { + Directory dir = _taxonomyDirectoryFactory(luceneIndex.Name + "taxonomy"); + if (forceUnlock) + { + IndexWriter.Unlock(dir); + } + return dir; + } } } diff --git a/src/Examine.Lucene/Directories/DirectoryFactoryBase.cs b/src/Examine.Lucene/Directories/DirectoryFactoryBase.cs index 043dc136a..3658d0b2b 100644 --- a/src/Examine.Lucene/Directories/DirectoryFactoryBase.cs +++ b/src/Examine.Lucene/Directories/DirectoryFactoryBase.cs @@ -15,9 +15,17 @@ Directory IDirectoryFactory.CreateDirectory(LuceneIndex luceneIndex, bool forceU luceneIndex.Name, s => CreateDirectory(luceneIndex, forceUnlock)); + Directory IDirectoryFactory.CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock) + => _createdDirectories.GetOrAdd( + luceneIndex.Name + "_taxonomy", + s => CreateTaxonomyDirectory(luceneIndex, forceUnlock)); + /// protected abstract Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock); + /// + protected abstract Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock); + /// protected virtual void Dispose(bool disposing) { diff --git a/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs index cf3bddb3f..f1274735c 100644 --- a/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs @@ -35,5 +35,19 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force } return dir; } + + /// + protected override Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock) + { + var path = Path.Combine(_baseDir.FullName, luceneIndex.Name,"taxonomy"); + var luceneIndexFolder = new DirectoryInfo(path); + + var dir = FSDirectory.Open(luceneIndexFolder, LockFactory.GetLockFactory(luceneIndexFolder)); + if (forceUnlock) + { + IndexWriter.Unlock(dir); + } + return dir; + } } } diff --git a/src/Examine.Lucene/Directories/IDirectoryFactory.cs b/src/Examine.Lucene/Directories/IDirectoryFactory.cs index 0fc056b91..c59228b10 100644 --- a/src/Examine.Lucene/Directories/IDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/IDirectoryFactory.cs @@ -23,5 +23,15 @@ public interface IDirectoryFactory : IDisposable /// Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock); + /// + /// Creates the directory instance for the Taxonomy Index + /// + /// + /// If true, will force unlock the directory when created + /// + /// + /// Any subsequent calls for the same index will return the same directory instance + /// + Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock); } } diff --git a/src/Examine.Lucene/Directories/SyncedTaxonomyFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/SyncedTaxonomyFileSystemDirectoryFactory.cs new file mode 100644 index 000000000..55a9b3778 --- /dev/null +++ b/src/Examine.Lucene/Directories/SyncedTaxonomyFileSystemDirectoryFactory.cs @@ -0,0 +1,122 @@ + +using System; +using System.IO; +using System.Threading; +using Examine.Lucene.Providers; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Index; +using Lucene.Net.Store; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Directory = Lucene.Net.Store.Directory; + +namespace Examine.Lucene.Directories +{ + /// + /// A directory factory that replicates the index and it's Taxonomy Index from main storage on initialization to another + /// directory, then creates a lucene Directory based on that replicated index. A replication thread + /// is spawned to then replicate the local index back to the main storage location. + /// + /// + /// By default, Examine configures the local directory to be the %temp% folder. + /// + public class SyncedTaxonomyFileSystemDirectoryFactory : FileSystemDirectoryFactory + { + private readonly DirectoryInfo _localDir; + private readonly ILoggerFactory _loggerFactory; + private ExamineTaxonomyReplicator? _replicator; + + /// + public SyncedTaxonomyFileSystemDirectoryFactory( + DirectoryInfo localDir, + DirectoryInfo mainDir, + ILockFactory lockFactory, + ILoggerFactory loggerFactory) + : base(mainDir, lockFactory) + { + _localDir = localDir; + _loggerFactory = loggerFactory; + } + + /// + protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) + { + var luceneTaxonomyIndex = luceneIndex as LuceneIndex; + var path = Path.Combine(_localDir.FullName, luceneIndex.Name); + var localLuceneIndexFolder = new DirectoryInfo(path); + + Directory mainDir = base.CreateDirectory(luceneIndex, forceUnlock); + + var taxonomyPath = Path.Combine(_localDir.FullName, luceneIndex.Name, "taxonomy"); + var localLuceneTaxonomyIndexFolder = new DirectoryInfo(taxonomyPath); + + Directory mainTaxonomyDir = base.CreateTaxonomyDirectory(luceneTaxonomyIndex, forceUnlock); + + // used by the replicator, will be a short lived directory for each synced revision and deleted when finished. + var tempDir = new DirectoryInfo(Path.Combine(_localDir.FullName, "Rep", Guid.NewGuid().ToString("N"))); + + if (DirectoryReader.IndexExists(mainDir) && DirectoryReader.IndexExists(mainTaxonomyDir)) + { + // when the lucene directory is going to be created, we'll sync from main storage to local + // storage before any index/writer is opened. + using (var tempMainIndexWriter = new IndexWriter( + mainDir, + new IndexWriterConfig( + LuceneInfo.CurrentVersion, + new StandardAnalyzer(LuceneInfo.CurrentVersion)) + { + OpenMode = OpenMode.APPEND, + IndexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()) + })) + using (var tempMainIndex = new LuceneIndex(_loggerFactory, luceneIndex.Name, new TempOptions(), tempMainIndexWriter)) + using (var tempLocalDirectory = new SimpleFSDirectory(localLuceneIndexFolder, LockFactory.GetLockFactory(localLuceneIndexFolder))) + using (var tempTaxonomyLocalDirectory = new SimpleFSDirectory(localLuceneTaxonomyIndexFolder, LockFactory.GetLockFactory(localLuceneTaxonomyIndexFolder))) + using (var replicator = new ExamineTaxonomyReplicator(_loggerFactory, tempMainIndex, tempLocalDirectory, tempTaxonomyLocalDirectory, tempDir)) + { + if (forceUnlock) + { + IndexWriter.Unlock(tempLocalDirectory); + } + + // replicate locally. + replicator.ReplicateIndex(); + } + } + + // now create the replicator that will copy from local to main on schedule + _replicator = new ExamineTaxonomyReplicator(_loggerFactory, luceneTaxonomyIndex, mainDir, mainTaxonomyDir, tempDir); + var localLuceneDir = FSDirectory.Open( + localLuceneIndexFolder, + LockFactory.GetLockFactory(localLuceneIndexFolder)); + + if (forceUnlock) + { + IndexWriter.Unlock(localLuceneDir); + } + + // Start replicating back to main + _replicator.StartIndexReplicationOnSchedule(1000); + + return localLuceneDir; + } + + /// + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + if (disposing) + { + _replicator?.Dispose(); + } + } + + private class TempOptions : IOptionsMonitor + { + public LuceneDirectoryIndexOptions CurrentValue => new LuceneDirectoryIndexOptions(); + public LuceneDirectoryIndexOptions Get(string name) => CurrentValue; + + public IDisposable OnChange(Action listener) => throw new NotImplementedException(); + } + + } +} diff --git a/src/Examine.Lucene/ExamineTaxonomyReplicator.cs b/src/Examine.Lucene/ExamineTaxonomyReplicator.cs new file mode 100644 index 000000000..574d118d0 --- /dev/null +++ b/src/Examine.Lucene/ExamineTaxonomyReplicator.cs @@ -0,0 +1,178 @@ +using System; +using System.IO; +using Examine.Lucene.Providers; +using Lucene.Net.Index; +using Lucene.Net.Replicator; +using Lucene.Net.Store; +using Microsoft.Extensions.Logging; +using static Lucene.Net.Replicator.IndexAndTaxonomyRevision; +using Directory = Lucene.Net.Store.Directory; + +namespace Examine.Lucene +{ + /// + /// Used to replicate an index to a destination directory + /// + /// + /// The destination directory must not have any active writers open to it. + /// + public class ExamineTaxonomyReplicator : IDisposable + { + private bool _disposedValue; + private readonly IReplicator _replicator; + private readonly LuceneIndex _sourceIndex; + private readonly Directory _destinationDirectory; + private readonly ReplicationClient _localReplicationClient; + private readonly object _locker = new object(); + private bool _started = false; + private readonly ILogger _logger; + + /// + /// Constructor + /// + /// + /// + /// + /// + /// + public ExamineTaxonomyReplicator( + ILoggerFactory loggerFactory, + LuceneIndex sourceIndex, + Directory destinationDirectory, + Directory destinationTaxonomyDirectory, + DirectoryInfo tempStorage) + { + _sourceIndex = sourceIndex; + _destinationDirectory = destinationDirectory; + _replicator = new LocalReplicator(); + _logger = loggerFactory.CreateLogger(); + + _localReplicationClient = new LoggingReplicationClient( + loggerFactory.CreateLogger(), + _replicator, + new IndexAndTaxonomyReplicationHandler( + destinationDirectory, + destinationTaxonomyDirectory, + () => + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + var sourceDir = sourceIndex.GetLuceneDirectory() as FSDirectory; + var destDir = destinationDirectory as FSDirectory; + + + var sourceTaxonomyDir = sourceIndex.GetLuceneTaxonomyDirectory() as FSDirectory; + var destTaxonomyDir = destinationTaxonomyDirectory as FSDirectory; + + // Callback, can be used to notifiy when replication is done (i.e. to open the index) + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "{IndexName} replication complete from {SourceDirectory} to {DestinationDirectory} and Taxonomy {TaxonomySourceDirectory} to {TaxonomyDestinationDirectory}", + sourceIndex.Name, + sourceDir?.Directory.ToString() ?? "InMemory", + destDir?.Directory.ToString() ?? "InMemory", + sourceTaxonomyDir?.Directory.ToString() ?? "InMemory", + destTaxonomyDir?.Directory.ToString() ?? "InMemory" + ); + } + } + + }), + new PerSessionDirectoryFactory(tempStorage.FullName)); + } + + /// + /// Will sync from the active index to the destination directory + /// + public void ReplicateIndex() + { + if (IndexWriter.IsLocked(_destinationDirectory)) + { + throw new InvalidOperationException("The destination directory is locked"); + } + + IndexAndTaxonomyRevision rev; + try + { + rev = new IndexAndTaxonomyRevision(_sourceIndex.IndexWriter.IndexWriter, _sourceIndex.TaxonomyWriter as SnapshotDirectoryTaxonomyWriter); + } + catch (InvalidOperationException) + { + // will occur if there is nothing to sync + return; + } + + _replicator.Publish(rev); + _localReplicationClient.UpdateNow(); + } + + /// + /// Starts a thread that will replicate the index on a schedule + /// + /// Interval + /// + public void StartIndexReplicationOnSchedule(int milliseconds) + { + lock (_locker) + { + if (_started) + { + return; + } + + _started = true; + + if (IndexWriter.IsLocked(_destinationDirectory)) + { + throw new InvalidOperationException("The destination directory is locked"); + } + + _sourceIndex.IndexCommitted += SourceIndex_IndexCommitted; + + // this will update the destination every second if there are changes. + // the change monitor will be stopped when this is disposed. + _localReplicationClient.StartUpdateThread(milliseconds, $"IndexRep{_sourceIndex.Name}"); + } + + } + + /// + /// Whenever the index is committed, publish the new revision to be synced. + /// + /// + /// + private void SourceIndex_IndexCommitted(object sender, EventArgs e) + { + var index = (LuceneIndex)sender; + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("{IndexName} committed", index.Name); + } + var rev = new IndexAndTaxonomyRevision(_sourceIndex.IndexWriter.IndexWriter, _sourceIndex.TaxonomyWriter as SnapshotDirectoryTaxonomyWriter); + _replicator.Publish(rev); + } + + /// + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + _sourceIndex.IndexCommitted -= SourceIndex_IndexCommitted; + _localReplicationClient.Dispose(); + } + + _disposedValue = true; + } + } + + /// + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + } + } +} diff --git a/src/Examine.Lucene/Indexing/DateTimeType.cs b/src/Examine.Lucene/Indexing/DateTimeType.cs index a04bada7c..f13d6cb49 100644 --- a/src/Examine.Lucene/Indexing/DateTimeType.cs +++ b/src/Examine.Lucene/Indexing/DateTimeType.cs @@ -22,6 +22,7 @@ public class DateTimeType : IndexFieldRangeValueType, IIndexFacetValue public DateResolution Resolution { get; } private readonly bool _isFacetable; + private readonly bool _taxonomyIndex; /// /// Can be sorted by the normal field name @@ -29,11 +30,15 @@ public class DateTimeType : IndexFieldRangeValueType, IIndexFacetValue public override string SortableFieldName => FieldName; /// - public DateTimeType(string fieldName, ILoggerFactory logger, DateResolution resolution, bool store, bool isFacetable) + public bool IsTaxonomyFaceted => _taxonomyIndex; + + /// + public DateTimeType(string fieldName, ILoggerFactory logger, DateResolution resolution, bool store, bool isFacetable, bool taxonomyIndex) : base(fieldName, logger, store) { Resolution = resolution; _isFacetable = isFacetable; + _taxonomyIndex = taxonomyIndex; } /// @@ -44,6 +49,27 @@ public DateTimeType(string fieldName, ILoggerFactory logger, DateResolution reso _isFacetable = false; } + public override void AddValue(Document doc, object value) + { + // Support setting taxonomy path + if (_isFacetable && _taxonomyIndex && value is object[] objArr && objArr != null && objArr.Length == 2) + { + if (!TryConvert(objArr[0], out DateTime parsedVal)) + return; + if (!TryConvert(objArr[1], out string[] parsedPathVal)) + return; + + var val = DateToLong(parsedVal); + + doc.Add(new Int64Field(FieldName, val, Store ? Field.Store.YES : Field.Store.NO)); + + doc.Add(new FacetField(FieldName, parsedPathVal)); + doc.Add(new NumericDocValuesField(FieldName, val)); + return; + } + base.AddValue(doc, value); + } + /// protected override void AddSingleValue(Document doc, object value) { @@ -54,7 +80,12 @@ protected override void AddSingleValue(Document doc, object value) doc.Add(new Int64Field(FieldName,val, Store ? Field.Store.YES : Field.Store.NO)); - if (_isFacetable) + if (_isFacetable && _taxonomyIndex) + { + doc.Add(new FacetField(FieldName, val.ToString())); + doc.Add(new NumericDocValuesField(FieldName, val)); + } + else if (_isFacetable && !_taxonomyIndex) { doc.Add(new SortedSetDocValuesFacetField(FieldName, val.ToString())); doc.Add(new NumericDocValuesField(FieldName, val)); diff --git a/src/Examine.Lucene/Indexing/DoubleType.cs b/src/Examine.Lucene/Indexing/DoubleType.cs index d8c55376f..c7afdaee6 100644 --- a/src/Examine.Lucene/Indexing/DoubleType.cs +++ b/src/Examine.Lucene/Indexing/DoubleType.cs @@ -16,12 +16,14 @@ namespace Examine.Lucene.Indexing public class DoubleType : IndexFieldRangeValueType, IIndexFacetValueType { private readonly bool _isFacetable; - + private readonly bool _taxonomyIndex; + /// - public DoubleType(string fieldName, ILoggerFactory logger, bool store, bool isFacetable) + public DoubleType(string fieldName, ILoggerFactory logger, bool store, bool isFacetable, bool taxonomyIndex = false) : base(fieldName, logger, store) { _isFacetable = isFacetable; + _taxonomyIndex = taxonomyIndex; } /// @@ -36,6 +38,29 @@ public DoubleType(string fieldName, ILoggerFactory logger, bool store = true) /// public override string SortableFieldName => FieldName; + /// + public bool IsTaxonomyFaceted => _taxonomyIndex; + + /// + public override void AddValue(Document doc, object value) + { + // Support setting taxonomy path + if (_isFacetable && _taxonomyIndex && value is object[] objArr && objArr != null && objArr.Length == 2) + { + if (!TryConvert(objArr[0], out double parsedVal)) + return; + if (!TryConvert(objArr[1], out string[] parsedPathVal)) + return; + + doc.Add(new DoubleField(FieldName, parsedVal, Store ? Field.Store.YES : Field.Store.NO)); + + doc.Add(new FacetField(FieldName, parsedPathVal)); + doc.Add(new DoubleDocValuesField(FieldName, parsedVal)); + return; + } + base.AddValue(doc, value); + } + /// protected override void AddSingleValue(Document doc, object value) { @@ -44,7 +69,12 @@ protected override void AddSingleValue(Document doc, object value) doc.Add(new DoubleField(FieldName,parsedVal, Store ? Field.Store.YES : Field.Store.NO)); - if (_isFacetable) + if (_isFacetable && _taxonomyIndex) + { + doc.Add(new FacetField(FieldName, parsedVal.ToString())); + doc.Add(new DoubleDocValuesField(FieldName, parsedVal)); + } + else if (_isFacetable && !_taxonomyIndex) { doc.Add(new SortedSetDocValuesFacetField(FieldName, parsedVal.ToString())); doc.Add(new DoubleDocValuesField(FieldName, parsedVal)); diff --git a/src/Examine.Lucene/Indexing/FullTextType.cs b/src/Examine.Lucene/Indexing/FullTextType.cs index 726189fc7..d0437e5db 100644 --- a/src/Examine.Lucene/Indexing/FullTextType.cs +++ b/src/Examine.Lucene/Indexing/FullTextType.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.IO; using Examine.Lucene.Analyzers; @@ -29,6 +30,7 @@ public class FullTextType : IndexFieldValueTypeBase, IIndexFacetValueType private readonly bool _sortable; private readonly Analyzer _analyzer; private readonly bool _isFacetable; + private readonly bool _taxonomyIndex; /// /// Constructor @@ -38,14 +40,16 @@ public class FullTextType : IndexFieldValueTypeBase, IIndexFacetValueType /// /// /// + /// /// Defaults to /// - public FullTextType(string fieldName, ILoggerFactory logger, bool sortable = false, bool isFacetable = false, Analyzer? analyzer = null) + public FullTextType(string fieldName, ILoggerFactory logger, bool sortable = false, bool isFacetable = false, Analyzer? analyzer = null, bool taxonomyIndex = false) : base(fieldName, logger, true) { _sortable = sortable; _analyzer = analyzer ?? new CultureInvariantStandardAnalyzer(); _isFacetable = isFacetable; + _taxonomyIndex = taxonomyIndex; } /// @@ -74,6 +78,36 @@ public FullTextType(string fieldName, ILoggerFactory logger, Analyzer? analyzer public override Analyzer Analyzer => _analyzer; /// + public bool IsTaxonomyFaceted => _taxonomyIndex; + + /// + public override void AddValue(Document doc, object value) + { + // Support setting taxonomy path + if (_isFacetable && _taxonomyIndex && value is object[] objArr && objArr != null && objArr.Length == 2) + { + if (!TryConvert(objArr[0], out string str)) + return; + if (!TryConvert(objArr[1], out string[] parsedPathVal)) + return; + + doc.Add(new TextField(FieldName, str, Field.Store.YES)); + + if (_sortable) + { + //to be sortable it cannot be analyzed so we have to make a different field + doc.Add(new StringField( + ExamineFieldNames.SortedFieldNamePrefix + FieldName, + str, + Field.Store.YES)); + } + + doc.Add(new FacetField(FieldName, parsedPathVal)); + return; + } + base.AddValue(doc, value); + } + protected override void AddSingleValue(Document doc, object value) { if (TryConvert(value, out var str)) @@ -89,7 +123,11 @@ protected override void AddSingleValue(Document doc, object value) Field.Store.YES)); } - if (_isFacetable) + if (_isFacetable && _taxonomyIndex) + { + doc.Add(new FacetField(FieldName, str)); + } + else if (_isFacetable && !_taxonomyIndex) { doc.Add(new SortedSetDocValuesFacetField(FieldName, str)); } diff --git a/src/Examine.Lucene/Indexing/IIndexFacetValueType.cs b/src/Examine.Lucene/Indexing/IIndexFacetValueType.cs index 1ecb8eb87..9b8d639e4 100644 --- a/src/Examine.Lucene/Indexing/IIndexFacetValueType.cs +++ b/src/Examine.Lucene/Indexing/IIndexFacetValueType.cs @@ -9,6 +9,11 @@ namespace Examine.Lucene.Indexing /// public interface IIndexFacetValueType { + /// + /// Whether the Field is indexed in the Taxonomy Index + /// + bool IsTaxonomyFaceted { get; } + /// /// Extracts the facets from the field /// diff --git a/src/Examine.Lucene/Indexing/Int32Type.cs b/src/Examine.Lucene/Indexing/Int32Type.cs index d3343949a..2521d9493 100644 --- a/src/Examine.Lucene/Indexing/Int32Type.cs +++ b/src/Examine.Lucene/Indexing/Int32Type.cs @@ -16,12 +16,14 @@ namespace Examine.Lucene.Indexing public class Int32Type : IndexFieldRangeValueType, IIndexFacetValueType { private readonly bool _isFacetable; + private readonly bool _taxonomyIndex; /// - public Int32Type(string fieldName, ILoggerFactory logger, bool store, bool isFacetable) + public Int32Type(string fieldName, ILoggerFactory logger, bool store, bool isFacetable, bool taxonomyIndex = false) : base(fieldName, logger, store) { _isFacetable = isFacetable; + _taxonomyIndex = taxonomyIndex; } /// @@ -37,14 +39,41 @@ public Int32Type(string fieldName, ILoggerFactory logger, bool store = true) public override string SortableFieldName => FieldName; /// + public bool IsTaxonomyFaceted => _taxonomyIndex; + + /// + public override void AddValue(Document doc, object value) + { + // Support setting taxonomy path + if (_isFacetable && _taxonomyIndex && value is object[] objArr && objArr != null && objArr.Length == 2) + { + if (!TryConvert(objArr[0], out int parsedVal)) + return; + if (!TryConvert(objArr[1], out string[] parsedPathVal)) + return; + + doc.Add(new Int32Field(FieldName, parsedVal, Store ? Field.Store.YES : Field.Store.NO)); + + doc.Add(new FacetField(FieldName, parsedPathVal)); + doc.Add(new NumericDocValuesField(FieldName, parsedVal)); + return; + } + base.AddValue(doc, value); + } + protected override void AddSingleValue(Document doc, object value) { if (!TryConvert(value, out int parsedVal)) return; - doc.Add(new Int32Field(FieldName,parsedVal, Store ? Field.Store.YES : Field.Store.NO)); + doc.Add(new Int32Field(FieldName, parsedVal, Store ? Field.Store.YES : Field.Store.NO)); - if (_isFacetable) + if (_isFacetable && _taxonomyIndex) + { + doc.Add(new FacetField(FieldName, parsedVal.ToString())); + doc.Add(new NumericDocValuesField(FieldName, parsedVal)); + } + else if (_isFacetable && !_taxonomyIndex) { doc.Add(new SortedSetDocValuesFacetField(FieldName, parsedVal.ToString())); doc.Add(new NumericDocValuesField(FieldName, parsedVal)); diff --git a/src/Examine.Lucene/Indexing/Int64Type.cs b/src/Examine.Lucene/Indexing/Int64Type.cs index c4b709360..524124155 100644 --- a/src/Examine.Lucene/Indexing/Int64Type.cs +++ b/src/Examine.Lucene/Indexing/Int64Type.cs @@ -16,12 +16,14 @@ namespace Examine.Lucene.Indexing public class Int64Type : IndexFieldRangeValueType, IIndexFacetValueType { private readonly bool _isFacetable; + private readonly bool _taxonomyIndex; /// - public Int64Type(string fieldName, ILoggerFactory logger, bool store, bool isFacetable) + public Int64Type(string fieldName, ILoggerFactory logger, bool store, bool isFacetable, bool taxonomyIndex = false) : base(fieldName, logger, store) { _isFacetable = isFacetable; + _taxonomyIndex = taxonomyIndex; } /// @@ -37,6 +39,28 @@ public Int64Type(string fieldName, ILoggerFactory logger, bool store = true) public override string SortableFieldName => FieldName; /// + public bool IsTaxonomyFaceted => _taxonomyIndex; + + /// + public override void AddValue(Document doc, object value) + { + // Support setting taxonomy path + if (_isFacetable && _taxonomyIndex && value is object[] objArr && objArr != null && objArr.Length == 2) + { + if (!TryConvert(objArr[0], out long parsedVal)) + return; + if (!TryConvert(objArr[1], out string[] parsedPathVal)) + return; + + doc.Add(new Int64Field(FieldName, parsedVal, Store ? Field.Store.YES : Field.Store.NO)); + + doc.Add(new FacetField(FieldName, parsedPathVal)); + doc.Add(new NumericDocValuesField(FieldName, parsedVal)); + return; + } + base.AddValue(doc, value); + } + protected override void AddSingleValue(Document doc, object value) { if (!TryConvert(value, out long parsedVal)) @@ -44,7 +68,12 @@ protected override void AddSingleValue(Document doc, object value) doc.Add(new Int64Field(FieldName, parsedVal, Store ? Field.Store.YES : Field.Store.NO)); - if (_isFacetable) + if (_isFacetable && _taxonomyIndex) + { + doc.Add(new FacetField(FieldName, parsedVal.ToString())); + doc.Add(new NumericDocValuesField(FieldName, parsedVal)); + } + else if (_isFacetable && !_taxonomyIndex) { doc.Add(new SortedSetDocValuesFacetField(FieldName, parsedVal.ToString())); doc.Add(new NumericDocValuesField(FieldName, parsedVal)); diff --git a/src/Examine.Lucene/Indexing/SingleType.cs b/src/Examine.Lucene/Indexing/SingleType.cs index f2d3c21b8..132877f2d 100644 --- a/src/Examine.Lucene/Indexing/SingleType.cs +++ b/src/Examine.Lucene/Indexing/SingleType.cs @@ -7,6 +7,7 @@ using Lucene.Net.Facet.SortedSet; using Lucene.Net.Search; using Microsoft.Extensions.Logging; +using static Lucene.Net.Queries.Function.ValueSources.MultiFunction; namespace Examine.Lucene.Indexing { @@ -16,12 +17,14 @@ namespace Examine.Lucene.Indexing public class SingleType : IndexFieldRangeValueType, IIndexFacetValueType { private readonly bool _isFacetable; + private readonly bool _taxonomyIndex; /// - public SingleType(string fieldName, ILoggerFactory logger, bool store, bool isFacetable) + public SingleType(string fieldName, ILoggerFactory logger, bool store, bool isFacetable, bool taxonomyIndex = false) : base(fieldName, logger, store) { _isFacetable = isFacetable; + _taxonomyIndex = taxonomyIndex; } /// @@ -37,14 +40,41 @@ public SingleType(string fieldName, ILoggerFactory logger, bool store = true) public override string SortableFieldName => FieldName; /// + public bool IsTaxonomyFaceted => _taxonomyIndex; + + /// + public override void AddValue(Document doc, object value) + { + // Support setting taxonomy path + if (_isFacetable && _taxonomyIndex && value is object[] objArr && objArr != null && objArr.Length == 2) + { + if (!TryConvert(objArr[0], out float parsedVal)) + return; + if (!TryConvert(objArr[1], out string[] parsedPathVal)) + return; + + doc.Add(new DoubleField(FieldName, parsedVal, Store ? Field.Store.YES : Field.Store.NO)); + + doc.Add(new FacetField(FieldName, parsedPathVal)); + doc.Add(new SingleDocValuesField(FieldName, parsedVal)); + return; + } + base.AddValue(doc, value); + } + protected override void AddSingleValue(Document doc, object value) { if (!TryConvert(value, out float parsedVal)) return; - doc.Add(new DoubleField(FieldName,parsedVal, Store ? Field.Store.YES : Field.Store.NO)); + doc.Add(new DoubleField(FieldName, parsedVal, Store ? Field.Store.YES : Field.Store.NO)); - if (_isFacetable) + if (_isFacetable && _taxonomyIndex) + { + doc.Add(new FacetField(FieldName, parsedVal.ToString())); + doc.Add(new SingleDocValuesField(FieldName, parsedVal)); + } + else if (_isFacetable && !_taxonomyIndex) { doc.Add(new SortedSetDocValuesFacetField(FieldName, parsedVal.ToString())); doc.Add(new SingleDocValuesField(FieldName, parsedVal)); diff --git a/src/Examine.Lucene/LuceneIndexOptions.cs b/src/Examine.Lucene/LuceneIndexOptions.cs index 7b41c1afe..5c136c27d 100644 --- a/src/Examine.Lucene/LuceneIndexOptions.cs +++ b/src/Examine.Lucene/LuceneIndexOptions.cs @@ -37,5 +37,10 @@ public class LuceneIndexOptions : IndexOptions /// This is generally used to initialize any custom value types for your indexer since the value type collection cannot be modified at runtime. /// public IReadOnlyDictionary? IndexValueTypesFactory { get; set; } + + /// + /// Gets or Sets whether to use a Taxonomy Index + /// + public bool UseTaxonomyIndex { get; set; } } } diff --git a/src/Examine.Lucene/Providers/BaseLuceneSearcher.cs b/src/Examine.Lucene/Providers/BaseLuceneSearcher.cs index 5b2a4c252..9b313f77b 100644 --- a/src/Examine.Lucene/Providers/BaseLuceneSearcher.cs +++ b/src/Examine.Lucene/Providers/BaseLuceneSearcher.cs @@ -10,7 +10,7 @@ namespace Examine.Lucene.Providers /// /// Simple abstract class containing basic properties for Lucene searchers /// - public abstract class BaseLuceneSearcher : BaseSearchProvider + public abstract class BaseLuceneSearcher : BaseSearchProvider, IDisposable { private readonly FacetsConfig _facetsConfig; @@ -67,6 +67,12 @@ public override ISearchResults Search(string searchText, QueryOptions? options = return sc.Execute(options); } + /// + public virtual void Dispose() + { + + } + ///// ///// This is NOT used! however I'm leaving this here as example code ///// diff --git a/src/Examine.Lucene/Providers/IIndexCommiter.cs b/src/Examine.Lucene/Providers/IIndexCommiter.cs new file mode 100644 index 000000000..eae249958 --- /dev/null +++ b/src/Examine.Lucene/Providers/IIndexCommiter.cs @@ -0,0 +1,20 @@ +using System; + +namespace Examine.Lucene.Providers +{ + /// + /// This queues up a commit for the index so that a commit doesn't happen on every individual write since that is quite expensive + /// + public interface IIndexCommiter : IDisposable + { + /// + /// Commits the index to directory + /// + void CommitNow(); + + /// + /// Schedules the index to be commited to the directory + /// + void ScheduleCommit(); + } +} diff --git a/src/Examine.Lucene/Providers/ILuceneTaxonomySearcher.cs b/src/Examine.Lucene/Providers/ILuceneTaxonomySearcher.cs new file mode 100644 index 000000000..1b25fdee0 --- /dev/null +++ b/src/Examine.Lucene/Providers/ILuceneTaxonomySearcher.cs @@ -0,0 +1,28 @@ +using System; +using Examine.Search; + +namespace Examine.Lucene.Providers +{ + /// + /// Lucene Taxonomy Searcher + /// + public interface ILuceneTaxonomySearcher : ISearcher, IDisposable + { + /// + /// The number of categories in the Taxonomy + /// + int CategoryCount { get; } + + /// + /// Returns the Ordinal for the dim and path + /// + int GetOrdinal(string dim, string[] path); + + /// + /// Given a dimensions ordinal (id), get the Path. + /// + /// Demension ordinal (id) + /// Path + IFacetLabel GetPath(int ordinal); + } +} diff --git a/src/Examine.Lucene/Providers/LuceneIndex.cs b/src/Examine.Lucene/Providers/LuceneIndex.cs index 1bddb52c8..a41e33106 100644 --- a/src/Examine.Lucene/Providers/LuceneIndex.cs +++ b/src/Examine.Lucene/Providers/LuceneIndex.cs @@ -18,6 +18,9 @@ using Lucene.Net.Analysis.Standard; using Examine.Lucene.Indexing; using Examine.Lucene.Directories; +using Lucene.Net.Facet.Taxonomy; +using Lucene.Net.Facet.Taxonomy.Directory; +using static Lucene.Net.Replicator.IndexAndTaxonomyRevision; namespace Examine.Lucene.Providers { @@ -31,6 +34,49 @@ public class LuceneIndex : BaseIndexProvider, IDisposable, IIndexStats, Referenc { #region Constructors + protected LuceneIndex( + ILoggerFactory loggerFactory, + string name, + IOptionsMonitor indexOptions, + Func indexCommiterFactory, + IndexWriter writer = null) + : base(loggerFactory, name, indexOptions) + { + _options = indexOptions.GetNamedOptions(name); + _committer = indexCommiterFactory(this); + _logger = loggerFactory.CreateLogger(); + + //initialize the field types + _fieldValueTypeCollection = new Lazy(() => CreateFieldValueTypes(_options.IndexValueTypesFactory)); + + _cancellationTokenSource = new CancellationTokenSource(); + _cancellationToken = _cancellationTokenSource.Token; + + if (writer != null) + { + _writer = new TrackingIndexWriter(writer ?? throw new ArgumentNullException(nameof(writer))); + DefaultAnalyzer = writer.Analyzer; + } + else + { + DefaultAnalyzer = _options.Analyzer ?? new StandardAnalyzer(LuceneInfo.CurrentVersion); + } + LuceneDirectoryIndexOptions directoryOptions = indexOptions.GetNamedOptions(name); + + if (directoryOptions.DirectoryFactory == null) + { + throw new InvalidOperationException($"No {typeof(IDirectoryFactory)} assigned"); + } + + if (_options.UseTaxonomyIndex) + { + _taxonomyDirectory = new Lazy(() => directoryOptions.DirectoryFactory.CreateTaxonomyDirectory(this, directoryOptions.UnlockIndex)); + } + + _directory = new Lazy(() => directoryOptions.DirectoryFactory.CreateDirectory(this, directoryOptions.UnlockIndex)); + } + + private LuceneIndex( ILoggerFactory loggerFactory, string name, @@ -44,7 +90,17 @@ private LuceneIndex( //initialize the field types _fieldValueTypeCollection = new Lazy(() => CreateFieldValueTypes(_options.IndexValueTypesFactory)); - _searcher = new Lazy(CreateSearcher); + if (_options.UseTaxonomyIndex) + { + _taxonomySearcher = new Lazy(CreateTaxonomySearcher); + + _searcher = new Lazy(() => _taxonomySearcher.Value); + } + else + { + _taxonomySearcher = new Lazy(() => throw new NotSupportedException("TaxonomySearcher not supported when not using taxonomy index.")); + _searcher = new Lazy(CreateSearcher); + } _cancellationTokenSource = new CancellationTokenSource(); _cancellationToken = _cancellationTokenSource.Token; @@ -61,13 +117,17 @@ public LuceneIndex( : this(loggerFactory, name, (IOptionsMonitor)indexOptions) { LuceneDirectoryIndexOptions directoryOptions = indexOptions.GetNamedOptions(name); - + if (directoryOptions.DirectoryFactory == null) { throw new InvalidOperationException($"No {typeof(IDirectoryFactory)} assigned"); } - _directory = new Lazy(() => directoryOptions.DirectoryFactory.CreateDirectory(this, directoryOptions.UnlockIndex)); + if (_options.UseTaxonomyIndex) + { + _taxonomyDirectory = new Lazy(() => directoryOptions.DirectoryFactory.CreateTaxonomyDirectory(this, directoryOptions.UnlockIndex)); + } + _directory = new Lazy(() => directoryOptions.DirectoryFactory.CreateDirectory(this, directoryOptions.UnlockIndex)); } //TODO: The problem with this is that the writer would already need to be configured with a PerFieldAnalyzerWrapper @@ -97,7 +157,7 @@ internal LuceneIndex( private FileStream? _logOutput; #endif private bool _disposedValue; - private readonly IndexCommiter _committer; + private readonly IIndexCommiter _committer; private volatile TrackingIndexWriter? _writer; @@ -113,15 +173,25 @@ internal LuceneIndex( /// private readonly object _writerLocker = new object(); - private readonly Lazy _searcher; + private readonly Lazy _searcher; private bool? _exists; + /// + /// Whether the Taxonomny Index exists. + /// + private bool? _taxonomyExists; + /// /// Gets a searcher for the index /// public override ISearcher Searcher => _searcher.Value; + /// + /// Gets a Taxonomy searcher for the index + /// + public virtual ILuceneTaxonomySearcher TaxonomySearcher => _taxonomySearcher.Value; + /// /// The async task that runs during an async indexing operation /// @@ -142,7 +212,13 @@ internal LuceneIndex( // tracks the latest Generation value of what has been indexed.This can be used to force update a searcher to this generation. private long? _latestGen; -#region Properties + private volatile DirectoryTaxonomyWriter _taxonomyWriter; + private ControlledRealTimeReopenThread _taxonomyNrtReopenThread; + + private readonly Lazy _taxonomySearcher; + private readonly Lazy _taxonomyDirectory; + + #region Properties /// /// Returns the configured for this index @@ -196,7 +272,12 @@ internal LuceneIndex( /// public event EventHandler? IndexCommitted; -#endregion + protected void RaiseIndexCommited(object sender, EventArgs e) + { + IndexCommitted?.Invoke(sender, e); + } + + #endregion #region Event handlers @@ -318,7 +399,7 @@ private int PerformIndexItemsInternal(IEnumerable valueSets, Cancellat /// public void EnsureIndex(bool forceOverwrite) { - if (!forceOverwrite && _exists.HasValue && _exists.Value) + if (!forceOverwrite && _exists.HasValue && _exists.Value && (!_options.UseTaxonomyIndex || (_taxonomyExists.HasValue && _taxonomyExists.Value))) { return; } @@ -332,10 +413,9 @@ public void EnsureIndex(bool forceOverwrite) { try { - var dir = GetLuceneDirectory(); - if (!indexExists) { + var dir = GetLuceneDirectory(); if (_logger.IsEnabled(LogLevel.Debug)) { _logger.LogDebug("Initializing new index {IndexName}", Name); @@ -343,6 +423,13 @@ public void EnsureIndex(bool forceOverwrite) //if there's no index, we need to create one CreateNewIndex(dir); + + if (_options.UseTaxonomyIndex) + { + //Now create the taxonomy index + var taxonomyDir = GetLuceneTaxonomyDirectory(); + CreateNewTaxonomyIndex(taxonomyDir); + } } else { @@ -362,6 +449,11 @@ public void EnsureIndex(bool forceOverwrite) _writer = CreateIndexWriterInternal(); } + if (_options.UseTaxonomyIndex && _taxonomyWriter == null) + { + _taxonomyWriter = CreateTaxonomyWriterInternal(); + } + //We're forcing an overwrite, // this means that we need to cancel all operations currently in place, // clear the queue and delete all of the data in the index. @@ -442,6 +534,33 @@ private void CreateNewIndex(Directory? dir) } } + /// + /// Used internally to create a brand new taxonomy index, this is not thread safe + /// + private void CreateNewTaxonomyIndex(Directory dir) + { + DirectoryTaxonomyWriter writer = null; + try + { + if (IsLocked(dir)) + { + //unlock it! + Unlock(dir); + } + //create the writer (this will overwrite old index files) + writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE); + } + catch (Exception ex) + { + OnIndexingError(new IndexingErrorEventArgs(this, "An error occurred creating the index", null, ex)); + return; + } + finally + { + writer?.Dispose(); + _taxonomyExists = true; + } + } /// /// Creates a new index, any existing index will be deleted @@ -587,7 +706,16 @@ protected virtual FieldValueTypeCollection CreateFieldValueTypes(IReadOnlyDictio /// Check if there is an index in the index folder /// /// - public override bool IndexExists() => _writer != null || IndexExistsImpl(); + public override bool IndexExists() + { + var mainIndexExists = _writer != null || IndexExistsImpl(); + if (!_options.UseTaxonomyIndex) + { + return mainIndexExists; + } + var taxonomyIndexExists = _taxonomyWriter != null || TaxonomyIndexExistsImpl(); + return taxonomyIndexExists && mainIndexExists; + } /// /// Check if the index is readable/healthy @@ -651,6 +779,30 @@ private bool IndexExistsImpl() return _exists.Value; } + // + /// This will check one time if the taxonomny index exists, we don't want to keep using IndexReader.IndexExists because that will literally go list + /// every file in the index folder and we don't need any more IO ops + /// + /// + /// + /// If the index does not exist, it will not store the value so subsequent calls to this will re-evaulate + /// + + private bool TaxonomyIndexExistsImpl() + { + //if it's been set and it's true, return true + if (_taxonomyExists.HasValue && _taxonomyExists.Value) + return true; + + //if it's not been set or it just doesn't exist, re-read the lucene files + if (!_taxonomyExists.HasValue || !_taxonomyExists.Value) + { + _taxonomyExists = DirectoryReader.IndexExists(GetLuceneTaxonomyDirectory()); + } + + return _taxonomyExists.Value; + } + /// @@ -690,7 +842,7 @@ private bool DeleteFromIndex(Term indexTerm, bool performCommit = true) return false; } } - + /// /// Collects the data for the fields and adds the document which is then committed into Lucene.Net's index /// @@ -771,13 +923,20 @@ protected virtual void AddDocument(Document doc, ValueSet valueSet) } // TODO: try/catch with OutOfMemoryException (see docs on UpdateDocument), though i've never seen this in real life - _latestGen = IndexWriter.UpdateDocument(new Term(ExamineFieldNames.ItemIdFieldName, valueSet.Id), _options.FacetsConfig.Build(doc)); + if (_options.UseTaxonomyIndex) + { + _latestGen = IndexWriter.UpdateDocument(new Term(ExamineFieldNames.ItemIdFieldName, valueSet.Id), _options.FacetsConfig.Build(TaxonomyWriter, doc)); + } + else + { + _latestGen = IndexWriter.UpdateDocument(new Term(ExamineFieldNames.ItemIdFieldName, valueSet.Id), _options.FacetsConfig.Build(doc)); + } } /// /// This queues up a commit for the index so that a commit doesn't happen on every individual write since that is quite expensive /// - private class IndexCommiter : DisposableObjectSlim + private class IndexCommiter : DisposableObjectSlim, IIndexCommiter { private readonly LuceneIndex _index; private DateTime _timestamp; @@ -790,17 +949,24 @@ private class IndexCommiter : DisposableObjectSlim /// private const int MaxWaitMilliseconds = 300000; + /// + /// Constructor + /// + /// Index to commit public IndexCommiter(LuceneIndex index) { _index = index; } + /// public void CommitNow() { + _index._taxonomyWriter?.Commit(); _index._writer?.IndexWriter?.Commit(); _index.IndexCommitted?.Invoke(_index, EventArgs.Empty); } + /// public void ScheduleCommit() { lock (_locker) @@ -912,6 +1078,13 @@ private bool ProcessQueueItem(IndexOperation item) /// public Directory? GetLuceneDirectory() => _writer != null ? _writer.IndexWriter.Directory : _directory?.Value; + /// + /// Returns the Lucene Directory used to store the taxonomy index + /// + /// + public Directory GetLuceneTaxonomyDirectory() => _taxonomyWriter != null ? _taxonomyWriter.Directory : _taxonomyDirectory.Value; + + /// /// Used to create an index writer - this is called in GetIndexWriter (and therefore, GetIndexWriter should not be overridden) /// @@ -1034,7 +1207,83 @@ public TrackingIndexWriter IndexWriter } } -#endregion + /// + /// Used to create an index writer - this is called in GetIndexWriter (and therefore, GetIndexWriter should not be overridden) + /// + /// + private DirectoryTaxonomyWriter CreateTaxonomyWriterInternal() + { + Directory dir = GetLuceneTaxonomyDirectory(); + + // Unfortunatley if the appdomain is taken down this will remain locked, so we can + // ensure that it's unlocked here in that case. + try + { + if (IsLocked(dir)) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("Forcing index {IndexName} to be unlocked since it was left in a locked state", Name); + } + //unlock it! + Unlock(dir); + } + } + catch (Exception ex) + { + OnIndexingError(new IndexingErrorEventArgs(this, "The index was locked and could not be unlocked", null, ex)); + return null; + } + + DirectoryTaxonomyWriter writer = CreateTaxonomyWriter(dir); + + return writer; + } + + /// + /// Method that creates the IndexWriter + /// + /// + /// + protected virtual DirectoryTaxonomyWriter CreateTaxonomyWriter(Directory d) + { + if (d == null) + { + throw new ArgumentNullException(nameof(d)); + } + var taxonomyWriter = new SnapshotDirectoryTaxonomyWriter(d); + + return taxonomyWriter; + } + + public DirectoryTaxonomyWriter TaxonomyWriter + { + get + { + EnsureIndex(false); + + if (_taxonomyWriter == null) + { + Monitor.Enter(_writerLocker); + try + { + if (_taxonomyWriter == null) + { + _taxonomyWriter = CreateTaxonomyWriterInternal(); + } + } + finally + { + Monitor.Exit(_writerLocker); + } + + } + + return _taxonomyWriter; + } + } + + #endregion #region Private @@ -1068,6 +1317,36 @@ private LuceneSearcher CreateSearcher() return new LuceneSearcher(name + "Searcher", searcherManager, FieldAnalyzer, FieldValueTypeCollection, _options.FacetsConfig); } + private LuceneTaxonomySearcher CreateTaxonomySearcher() + { + var possibleSuffixes = new[] { "Index", "Indexer" }; + var name = Name; + foreach (var suffix in possibleSuffixes) + { + //trim the "Indexer" / "Index" suffix if it exists + if (!name.EndsWith(suffix)) + continue; + name = name.Substring(0, name.LastIndexOf(suffix, StringComparison.Ordinal)); + } + + TrackingIndexWriter writer = IndexWriter; + DirectoryTaxonomyWriter taxonomyWriter = TaxonomyWriter; + var searcherManager = new SearcherTaxonomyManager(writer.IndexWriter, true, new SearcherFactory(), taxonomyWriter); + searcherManager.AddListener(this); + _taxonomyNrtReopenThread = new ControlledRealTimeReopenThread(writer, searcherManager, 5.0, 1.0) + { + Name = $"{Name} Taxonomy NRT Reopen Thread", + IsBackground = true + }; + + _taxonomyNrtReopenThread.Start(); + + // wait for most recent changes when first creating the searcher + WaitForChanges(); + + return new LuceneTaxonomySearcher(name + "Searcher", searcherManager, FieldAnalyzer, FieldValueTypeCollection, _options.FacetsConfig); + } + /// /// Deletes the item from the index either by id or by category /// @@ -1279,7 +1558,13 @@ protected virtual void Dispose(bool disposing) _nrtReopenThread.Dispose(); } - if (_searcher.IsValueCreated) + if (_taxonomyNrtReopenThread != null) + { + _taxonomyNrtReopenThread.Interrupt(); + _taxonomyNrtReopenThread.Dispose(); + } + + if (_searcher != null && _searcher.IsValueCreated) { _searcher.Value.Dispose(); } @@ -1323,6 +1608,18 @@ protected virtual void Dispose(bool disposing) } + if (_taxonomyWriter != null) + { + try + { + // Taxonomy writer must be disposed before index writer + _taxonomyWriter?.Dispose(); + } + catch (Exception e) + { + OnIndexingError(new IndexingErrorEventArgs(this, "Error closing the Taxonomy index", "-1", e)); + } + } _cancellationTokenSource.Dispose(); diff --git a/src/Examine.Lucene/Providers/LuceneTaxonomySearcher.cs b/src/Examine.Lucene/Providers/LuceneTaxonomySearcher.cs new file mode 100644 index 000000000..08e13fc44 --- /dev/null +++ b/src/Examine.Lucene/Providers/LuceneTaxonomySearcher.cs @@ -0,0 +1,91 @@ +using System; +using Examine.Lucene.Search; +using Examine.Search; +using Lucene.Net.Analysis; +using Lucene.Net.Facet; +using Lucene.Net.Facet.Taxonomy; + +namespace Examine.Lucene.Providers +{ + public class LuceneTaxonomySearcher : BaseLuceneSearcher, IDisposable, ILuceneTaxonomySearcher + { + private readonly SearcherTaxonomyManager _searcherManager; + private readonly FieldValueTypeCollection _fieldValueTypeCollection; + private bool _disposedValue; + + /// + /// Constructor allowing for creating a NRT instance based on a given writer + /// + /// + /// + /// + /// + /// + public LuceneTaxonomySearcher(string name, SearcherTaxonomyManager searcherManager, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection, FacetsConfig facetsConfig) + : base(name, analyzer, facetsConfig) + { + _searcherManager = searcherManager; + _fieldValueTypeCollection = fieldValueTypeCollection; + } + + /// + public override ISearchContext GetSearchContext() + => new TaxonomySearchContext(_searcherManager, _fieldValueTypeCollection); + + /// + /// Gets the Taxonomy SearchContext + /// + /// + public virtual ITaxonomySearchContext GetTaxonomySearchContext() + => new TaxonomySearchContext(_searcherManager, _fieldValueTypeCollection); + + /// + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + _searcherManager.Dispose(); + } + + _disposedValue = true; + } + } + + /// + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + } + + /// + public int CategoryCount + { + get + { + var taxonomyReader = GetTaxonomySearchContext().GetTaxonomyAndSearcher().TaxonomyReader; + return taxonomyReader.Count; + } + } + + /// + public int GetOrdinal(string dimension, string[] path) + { + var taxonomyReader = GetTaxonomySearchContext().GetTaxonomyAndSearcher().TaxonomyReader; + return taxonomyReader.GetOrdinal(dimension, path); + } + + + /// + public IFacetLabel GetPath(int ordinal) + { + var taxonomyReader = GetTaxonomySearchContext().GetTaxonomyAndSearcher().TaxonomyReader; + var facetLabel = taxonomyReader.GetPath(ordinal); + var examineFacetLabel = new LuceneFacetLabel(facetLabel); + return examineFacetLabel; + } + } +} + diff --git a/src/Examine.Lucene/Search/FacetFullTextField.cs b/src/Examine.Lucene/Search/FacetFullTextField.cs index eebd86cea..055e0f4da 100644 --- a/src/Examine.Lucene/Search/FacetFullTextField.cs +++ b/src/Examine.Lucene/Search/FacetFullTextField.cs @@ -48,7 +48,7 @@ public FacetFullTextField(string field, string[] values, string facetField, int /// public IEnumerable> ExtractFacets(IFacetExtractionContext facetExtractionContext) { - Facets facetCounts = facetExtractionContext.GetFacetCounts(FacetField, false); + Facets facetCounts = facetExtractionContext.GetFacetCounts(FacetField, IsTaxonomyIndexed); if (Values != null && Values.Length > 0) { diff --git a/src/Examine.Lucene/Search/FacetLongField.cs b/src/Examine.Lucene/Search/FacetLongField.cs index 3b31f1fe8..1316ab197 100644 --- a/src/Examine.Lucene/Search/FacetLongField.cs +++ b/src/Examine.Lucene/Search/FacetLongField.cs @@ -1,9 +1,7 @@ using System.Collections.Generic; using System.Linq; using Examine.Search; -using Lucene.Net.Facet; using Lucene.Net.Facet.Range; -using Lucene.Net.Facet.SortedSet; namespace Examine.Lucene.Search { diff --git a/src/Examine.Lucene/Search/ILuceneSearchResults.cs b/src/Examine.Lucene/Search/ILuceneSearchResults.cs new file mode 100644 index 000000000..1f45ea1da --- /dev/null +++ b/src/Examine.Lucene/Search/ILuceneSearchResults.cs @@ -0,0 +1,19 @@ +namespace Examine.Lucene.Search +{ + /// + /// Lucene.NET Search Results + /// + public interface ILuceneSearchResults : ISearchResults + { + /// + /// Options for Searching After. Used for efficent deep paging. + /// + SearchAfterOptions SearchAfter { get; } + + /// + /// Returns the maximum score value encountered. Note that in case + /// scores are not tracked, this returns . + /// + float MaxScore { get; } + } +} diff --git a/src/Examine.Lucene/Search/ITaxonomySearchContext.cs b/src/Examine.Lucene/Search/ITaxonomySearchContext.cs new file mode 100644 index 000000000..1e954a389 --- /dev/null +++ b/src/Examine.Lucene/Search/ITaxonomySearchContext.cs @@ -0,0 +1,14 @@ +namespace Examine.Lucene.Search +{ + /// + /// Search Context for Taxonomy Searcher + /// + public interface ITaxonomySearchContext : ISearchContext + { + /// + /// Gets the Search and Taxonomny Reader reference + /// + /// + ITaxonomySearcherReference GetTaxonomyAndSearcher(); + } +} diff --git a/src/Examine.Lucene/Search/ITaxonomySearcherReference.cs b/src/Examine.Lucene/Search/ITaxonomySearcherReference.cs new file mode 100644 index 000000000..94b839d7f --- /dev/null +++ b/src/Examine.Lucene/Search/ITaxonomySearcherReference.cs @@ -0,0 +1,15 @@ +using Lucene.Net.Facet.Taxonomy.Directory; + +namespace Examine.Lucene.Search +{ + /// + /// Represents a Taxonomy Searcher Reference + /// + public interface ITaxonomySearcherReference : ISearcherReference + { + /// + /// Taxonomy Reader for the sidecar taxonomy index + /// + DirectoryTaxonomyReader TaxonomyReader { get; } + } +} diff --git a/src/Examine.Lucene/Search/LuceneFacetExtractionContext.cs b/src/Examine.Lucene/Search/LuceneFacetExtractionContext.cs index c45d70ed7..745f0aedf 100644 --- a/src/Examine.Lucene/Search/LuceneFacetExtractionContext.cs +++ b/src/Examine.Lucene/Search/LuceneFacetExtractionContext.cs @@ -1,6 +1,7 @@ -using Lucene.Net.Facet.SortedSet; -using Lucene.Net.Facet; using System; +using Lucene.Net.Facet; +using Lucene.Net.Facet.SortedSet; +using Lucene.Net.Facet.Taxonomy; namespace Examine.Lucene.Search { @@ -32,7 +33,11 @@ public virtual Facets GetFacetCounts(string facetIndexFieldName, bool isTaxonomy { if (isTaxonomyIndexed) { - throw new NotSupportedException("Taxonomy Index not supported"); + if (SearcherReference is ITaxonomySearcherReference taxonomySearcher) + { + return new FastTaxonomyFacetCounts(facetIndexFieldName, taxonomySearcher.TaxonomyReader, FacetConfig, FacetsCollector); + } + throw new InvalidOperationException("Cannot get FastTaxonomyFacetCounts for field not stored in the Taxonomy index"); } else { diff --git a/src/Examine.Lucene/Search/LuceneFacetLabel.cs b/src/Examine.Lucene/Search/LuceneFacetLabel.cs new file mode 100644 index 000000000..a81878423 --- /dev/null +++ b/src/Examine.Lucene/Search/LuceneFacetLabel.cs @@ -0,0 +1,33 @@ +using Lucene.Net.Facet.Taxonomy; + +namespace Examine.Lucene.Search +{ + /// + /// Lucene Facet Label + /// + public class LuceneFacetLabel : Examine.Search.IFacetLabel + { + private readonly FacetLabel _facetLabel; + + /// + /// Constructor + /// + /// Lucene Facet Label + public LuceneFacetLabel(FacetLabel facetLabel) + { + _facetLabel = facetLabel; + } + + /// + public string[] Components => _facetLabel.Components; + + /// + public int Length => _facetLabel.Length; + + /// + public int CompareTo(Examine.Search.IFacetLabel other) => _facetLabel.CompareTo(new FacetLabel(other.Components)); + + /// + public Examine.Search.IFacetLabel Subpath(int length) => new LuceneFacetLabel(_facetLabel.Subpath(length)); + } +} diff --git a/src/Examine.Lucene/Search/LuceneFacetSamplingQueryOptions.cs b/src/Examine.Lucene/Search/LuceneFacetSamplingQueryOptions.cs new file mode 100644 index 000000000..d001b4f22 --- /dev/null +++ b/src/Examine.Lucene/Search/LuceneFacetSamplingQueryOptions.cs @@ -0,0 +1,52 @@ +namespace Examine.Lucene.Search +{ + /// + /// Options for Lucene Facet Sampling + /// + public class LuceneFacetSamplingQueryOptions + { + /// + /// Constructor + /// + /// The preferred sample size. If the number of hits is greater than + /// the size, sampling will be done using a sample ratio of sampling + /// size / totalN. For example: 1000 hits, sample size = 10 results in + /// samplingRatio of 0.01. If the number of hits is lower, no sampling + /// is done at all + /// The random seed. If 0 then a seed will be chosen for you. + public LuceneFacetSamplingQueryOptions(int sampleSize, long seed) + { + SampleSize = sampleSize; + Seed = seed; + } + + /// + /// Constructor + /// + /// The preferred sample size. If the number of hits is greater than + /// the size, sampling will be done using a sample ratio of sampling + /// size / totalN. For example: 1000 hits, sample size = 10 results in + /// samplingRatio of 0.01. If the number of hits is lower, no sampling + /// is done at all + public LuceneFacetSamplingQueryOptions(int sampleSize) + { + SampleSize = sampleSize; + Seed = 0; + } + + /// + /// The preferred sample size. If the number of hits is greater than + /// the size, sampling will be done using a sample ratio of sampling + /// size / totalN. For example: 1000 hits, sample size = 10 results in + /// samplingRatio of 0.01. If the number of hits is lower, no sampling + /// is done at all + /// + public int SampleSize { get; } + + /// + /// The random seed. If 0 then a seed will be chosen for you. + /// + public long Seed { get; } + + } +} diff --git a/src/Examine.Lucene/Search/LuceneQueryOptions.cs b/src/Examine.Lucene/Search/LuceneQueryOptions.cs new file mode 100644 index 000000000..c941c37cb --- /dev/null +++ b/src/Examine.Lucene/Search/LuceneQueryOptions.cs @@ -0,0 +1,52 @@ +using System; +using Examine.Search; + +namespace Examine.Lucene.Search +{ + /// + /// Lucene.NET specific query options + /// + public class LuceneQueryOptions : QueryOptions + { + /// + /// Constructor + /// + /// Number of result documents to skip. + /// Optional number of result documents to take. + /// Optionally skip to results after the results from the previous search execution. Used for efficent deep paging. + /// Whether to track the maximum document score. For best performance, if not needed, leave false. + /// Whether to Track Document Scores. For best performance, if not needed, leave false. + /// Whether to apply Facet sampling to improve performance. If not required, leave null + public LuceneQueryOptions(int skip, int? take = null, SearchAfterOptions? searchAfter = null, bool trackDocumentScores = false, bool trackDocumentMaxScore = false, LuceneFacetSamplingQueryOptions? facetSampling = null) + : base(skip, take) + { + TrackDocumentScores = trackDocumentScores; + TrackDocumentMaxScore = trackDocumentMaxScore; + SearchAfter = searchAfter; + FacetRandomSampling = facetSampling; + } + + /// + /// Whether to Track Document Scores. For best performance, if not needed, leave false. + /// + public bool TrackDocumentScores { get; } + + /// + /// Whether to track the maximum document score. For best performance, if not needed, leave false. + /// + public bool TrackDocumentMaxScore { get; } + + /// + /// Options for Searching After. Used for efficent deep paging. + /// + public SearchAfterOptions? SearchAfter { get; } + + /// + /// Options for Lucene Facet Sampling. If not set, no Facet Sampling is applied. + /// + /// + /// Performance optimization for large sets + /// + public LuceneFacetSamplingQueryOptions? FacetRandomSampling { get; } + } +} diff --git a/src/Examine.Lucene/Search/LuceneSearchExecutor.cs b/src/Examine.Lucene/Search/LuceneSearchExecutor.cs index b74208d87..9e59f35f0 100644 --- a/src/Examine.Lucene/Search/LuceneSearchExecutor.cs +++ b/src/Examine.Lucene/Search/LuceneSearchExecutor.cs @@ -6,8 +6,10 @@ using Lucene.Net.Documents; using Lucene.Net.Facet; using Lucene.Net.Facet.SortedSet; +using Lucene.Net.Facet.Taxonomy; using Lucene.Net.Index; using Lucene.Net.Search; +using LuceneFacetResult = Lucene.Net.Facet.FacetResult; namespace Examine.Lucene.Search { @@ -18,17 +20,20 @@ namespace Examine.Lucene.Search public class LuceneSearchExecutor { private readonly QueryOptions _options; + private readonly LuceneQueryOptions _luceneQueryOptions; private readonly IEnumerable _sortField; private readonly ISearchContext _searchContext; private readonly Query _luceneQuery; private readonly ISet? _fieldsToLoad; private readonly IEnumerable _facetFields; - private int? _maxDoc; private readonly FacetsConfig _facetsConfig; + private int? _maxDoc; - internal LuceneSearchExecutor(QueryOptions? options, Query query, IEnumerable sortField, ISearchContext searchContext, ISet? fieldsToLoad, IEnumerable facetFields, FacetsConfig facetsConfig) + internal LuceneSearchExecutor(QueryOptions? options, Query query, IEnumerable sortField, ISearchContext searchContext, + ISet fieldsToLoad, IEnumerable facetFields, FacetsConfig facetsConfig) { _options = options ?? QueryOptions.Default; + _luceneQueryOptions = _options as LuceneQueryOptions; _luceneQuery = query ?? throw new ArgumentNullException(nameof(query)); _fieldsToLoad = fieldsToLoad; _sortField = sortField ?? throw new ArgumentNullException(nameof(sortField)); @@ -89,61 +94,145 @@ public ISearchResults Execute() var maxResults = Math.Min((_options.Skip + 1) * _options.Take, MaxDoc); maxResults = maxResults >= 1 ? maxResults : QueryOptions.DefaultMaxResults; + int numHits = maxResults; - ICollector topDocsCollector; SortField[] sortFields = _sortField as SortField[] ?? _sortField.ToArray(); - if (sortFields.Length > 0) - { - topDocsCollector = TopFieldCollector.Create( - new Sort(sortFields), maxResults, false, false, false, false); - } - else - { - topDocsCollector = TopScoreDocCollector.Create(maxResults, true); - } + Sort sort = null; + FieldDoc scoreDocAfter = null; + Filter filter = null; using (ISearcherReference searcher = _searchContext.GetSearcher()) { - FacetsCollector? facetsCollector; - if(_facetFields.Any()) + if (sortFields.Length > 0) { - facetsCollector = new FacetsCollector(); - searcher.IndexSearcher.Search(_luceneQuery, MultiCollector.Wrap(topDocsCollector, facetsCollector)); + sort = new Sort(sortFields); + sort.Rewrite(searcher.IndexSearcher); } - else + if (_luceneQueryOptions != null && _luceneQueryOptions.SearchAfter != null) { - facetsCollector = null; - searcher.IndexSearcher.Search(_luceneQuery, topDocsCollector); + //The document to find results after. + scoreDocAfter = GetScoreDocAfter(_luceneQueryOptions); + + // We want to only collect only the actual number of hits we want to take after the last document. We don't need to collect all previous/next docs. + numHits = _options.Take >= 1 ? _options.Take : QueryOptions.DefaultMaxResults; } TopDocs topDocs; + ICollector topDocsCollector; + bool trackMaxScore = _luceneQueryOptions == null ? false : _luceneQueryOptions.TrackDocumentMaxScore; + bool trackDocScores = _luceneQueryOptions == null ? false : _luceneQueryOptions.TrackDocumentScores; + if (sortFields.Length > 0) { - topDocs = ((TopFieldCollector)topDocsCollector).GetTopDocs(_options.Skip, _options.Take); + bool fillFields = true; + topDocsCollector = TopFieldCollector.Create(sort, numHits, scoreDocAfter, fillFields, trackDocScores, trackMaxScore, false); + } + else + { + topDocsCollector = TopScoreDocCollector.Create(numHits, scoreDocAfter, true); + } + FacetsCollector facetsCollector = null; + if (_facetFields.Any() && _luceneQueryOptions != null && _luceneQueryOptions.FacetRandomSampling != null) + { + var facetsCollectors = new RandomSamplingFacetsCollector(_luceneQueryOptions.FacetRandomSampling.SampleSize, _luceneQueryOptions.FacetRandomSampling.Seed); + } + else if (_facetFields.Any()) + { + facetsCollector = new FacetsCollector(); + } + + if (scoreDocAfter != null && sort != null) + { + if (facetsCollector != null) + { + topDocs = FacetsCollector.SearchAfter(searcher.IndexSearcher, scoreDocAfter, _luceneQuery, filter, _options.Take, sort, MultiCollector.Wrap(topDocsCollector, facetsCollector)); + } + else + { + topDocs = searcher.IndexSearcher.SearchAfter(scoreDocAfter, _luceneQuery, filter, _options.Take, sort, trackDocScores, trackMaxScore); + } + } + else if (scoreDocAfter != null && sort == null) + { + if (facetsCollector != null) + { + topDocs = facetsCollector.SearchAfter(searcher.IndexSearcher, scoreDocAfter, _luceneQuery, _options.Take, MultiCollector.Wrap(topDocsCollector, facetsCollector)); + } + else + { + topDocs = searcher.IndexSearcher.SearchAfter(scoreDocAfter, _luceneQuery, _options.Take); + } } else { - topDocs = ((TopScoreDocCollector)topDocsCollector).GetTopDocs(_options.Skip, _options.Take); + searcher.IndexSearcher.Search(_luceneQuery, MultiCollector.Wrap(topDocsCollector, facetsCollector)); + if (sortFields.Length > 0) + { + topDocs = ((TopFieldCollector)topDocsCollector).GetTopDocs(_options.Skip, _options.Take); + } + else + { + topDocs = ((TopScoreDocCollector)topDocsCollector).GetTopDocs(_options.Skip, _options.Take); + } } var totalItemCount = topDocs.TotalHits; - var results = new List(); + var results = new List(topDocs.ScoreDocs.Length); for (int i = 0; i < topDocs.ScoreDocs.Length; i++) { var result = GetSearchResult(i, topDocs, searcher.IndexSearcher); - if(result != null) + if (result != null) { results.Add(result); } } - + var searchAfterOptions = GetSearchAfterOptions(topDocs); + float maxScore = topDocs.MaxScore; var facets = ExtractFacets(facetsCollector, searcher); - return new LuceneSearchResults(results, totalItemCount, facets); + return new LuceneSearchResults(results, totalItemCount, facets, maxScore, searchAfterOptions); } } + private static FieldDoc GetScoreDocAfter(LuceneQueryOptions luceneQueryOptions) + { + FieldDoc scoreDocAfter; + var searchAfter = luceneQueryOptions.SearchAfter; + + object[] searchAfterSortFields = new object[0]; + if (luceneQueryOptions.SearchAfter.Fields != null && luceneQueryOptions.SearchAfter.Fields.Length > 0) + { + searchAfterSortFields = luceneQueryOptions.SearchAfter.Fields; + } + if (searchAfter.ShardIndex != null) + { + scoreDocAfter = new FieldDoc(searchAfter.DocumentId, searchAfter.DocumentScore, searchAfterSortFields, searchAfter.ShardIndex.Value); + } + else + { + scoreDocAfter = new FieldDoc(searchAfter.DocumentId, searchAfter.DocumentScore, searchAfterSortFields); + } + + return scoreDocAfter; + } + + private static SearchAfterOptions GetSearchAfterOptions(TopDocs topDocs) + { + if (topDocs.TotalHits > 0) + { + if (topDocs.ScoreDocs.LastOrDefault() is FieldDoc lastFieldDoc && lastFieldDoc != null) + { + return new SearchAfterOptions(lastFieldDoc.Doc, lastFieldDoc.Score, lastFieldDoc.Fields?.ToArray(), lastFieldDoc.ShardIndex); + } + if (topDocs.ScoreDocs.LastOrDefault() is ScoreDoc scoreDoc && scoreDoc != null) + { + return new SearchAfterOptions(scoreDoc.Doc, scoreDoc.Score, new object[0], scoreDoc.ShardIndex); + } + } + return null; + } + private IReadOnlyDictionary ExtractFacets(FacetsCollector? facetsCollector, ISearcherReference searcher) { var facets = new Dictionary(StringComparer.InvariantCultureIgnoreCase); @@ -154,12 +243,14 @@ private IReadOnlyDictionary ExtractFacets(FacetsCollector? var facetFields = _facetFields.OrderBy(field => field.FacetField); - SortedSetDocValuesReaderState? sortedSetReaderState = null; + SortedSetDocValuesReaderState sortedSetReaderState = null; + Facets fastTaxonomyFacetCounts = null; - foreach(var field in facetFields) + + foreach (var field in facetFields) { var valueType = _searchContext.GetFieldValueType(field.Field); - if(valueType is IIndexFacetValueType facetValueType) + if (valueType is IIndexFacetValueType facetValueType) { var facetExtractionContext = new LuceneFacetExtractionContext(facetsCollector, searcher, _facetsConfig); @@ -175,7 +266,7 @@ private IReadOnlyDictionary ExtractFacets(FacetsCollector? return facets; } - private ISearchResult? GetSearchResult(int index, TopDocs topDocs, IndexSearcher luceneSearcher) + private LuceneSearchResult? GetSearchResult(int index, TopDocs topDocs, IndexSearcher luceneSearcher) { // I have seen IndexOutOfRangeException here which is strange as this is only called in one place // and from that one place "i" is always less than the size of this collection. @@ -198,8 +289,8 @@ private IReadOnlyDictionary ExtractFacets(FacetsCollector? doc = luceneSearcher.Doc(docId); } var score = scoreDoc.Score; - var result = CreateSearchResult(doc, score); - + var shardIndex = scoreDoc.ShardIndex; + var result = CreateSearchResult(doc, score, shardIndex); return result; } @@ -209,7 +300,7 @@ private IReadOnlyDictionary ExtractFacets(FacetsCollector? /// The doc to convert. /// The score. /// A populated search result object - private ISearchResult CreateSearchResult(Document doc, float score) + private LuceneSearchResult CreateSearchResult(Document doc, float score, int shardIndex) { var id = doc.Get("id"); @@ -218,7 +309,7 @@ private ISearchResult CreateSearchResult(Document doc, float score) id = doc.Get(ExamineFieldNames.ItemIdFieldName); } - var searchResult = new SearchResult(id, score, () => + var searchResult = new LuceneSearchResult(id, score, () => { //we can use lucene to find out the fields which have been stored for this particular document var fields = doc.Fields; @@ -247,7 +338,7 @@ private ISearchResult CreateSearchResult(Document doc, float score) } return resultVals; - }); + }, shardIndex); return searchResult; } diff --git a/src/Examine.Lucene/Search/LuceneSearchExtensions.cs b/src/Examine.Lucene/Search/LuceneSearchExtensions.cs index 37decee14..b3028fbf9 100644 --- a/src/Examine.Lucene/Search/LuceneSearchExtensions.cs +++ b/src/Examine.Lucene/Search/LuceneSearchExtensions.cs @@ -1,4 +1,4 @@ -using System; +using System; using Examine.Search; using Lucene.Net.Search; @@ -50,5 +50,17 @@ public static BooleanOperation ToBooleanOperation(this Occur o) return BooleanOperation.Or; } } + /// + /// Executes the query + /// + public static ILuceneSearchResults ExecuteWithLucene(this IQueryExecutor queryExecutor, QueryOptions options = null) + { + var results = queryExecutor.Execute(options); + if (results is ILuceneSearchResults luceneSearchResults) + { + return luceneSearchResults; + } + throw new NotSupportedException("QueryExecutor is not Lucene.NET"); + } } } diff --git a/src/Examine.Lucene/Search/LuceneSearchQuery.cs b/src/Examine.Lucene/Search/LuceneSearchQuery.cs index 01b01df4b..7047fe8f8 100644 --- a/src/Examine.Lucene/Search/LuceneSearchQuery.cs +++ b/src/Examine.Lucene/Search/LuceneSearchQuery.cs @@ -248,7 +248,7 @@ private ISearchResults Search(QueryOptions? options) } } - var executor = new LuceneSearchExecutor(options, query, SortFields, _searchContext, _fieldsToLoad, _facetFields, _facetsConfig); + var executor = new LuceneSearchExecutor(options, query, SortFields, _searchContext, _fieldsToLoad, _facetFields,_facetsConfig); var pagesResults = executor.Execute(); @@ -344,8 +344,10 @@ internal IFacetOperations FacetInternal(string field, Action? { values = Array.Empty(); } - var fieldValueType = _searchContext.GetFieldValueType(field); - var facet = new FacetFullTextField(field, values, GetFacetField(field)); + + var valueType = _searchContext.GetFieldValueType(field) as IIndexFacetValueType; + + var facet = new FacetFullTextField(field, values, GetFacetField(field), isTaxonomyIndexed: valueType.IsTaxonomyFaceted); if(facetConfiguration != null) { @@ -364,7 +366,8 @@ internal IFacetOperations FacetInternal(string field, params DoubleRange[] doubl doubleRanges = Array.Empty(); } - var facet = new FacetDoubleField(field, doubleRanges, GetFacetField(field)); + var valueType = _searchContext.GetFieldValueType(field) as IIndexFacetValueType; + var facet = new FacetDoubleField(field, doubleRanges, GetFacetField(field), isTaxonomyIndexed: valueType.IsTaxonomyFaceted); _facetFields.Add(facet); @@ -378,7 +381,8 @@ internal IFacetOperations FacetInternal(string field, params FloatRange[] floatR floatRanges = Array.Empty(); } - var facet = new FacetFloatField(field, floatRanges, GetFacetField(field)); + var valueType = _searchContext.GetFieldValueType(field) as IIndexFacetValueType; + var facet = new FacetFloatField(field, floatRanges, GetFacetField(field), isTaxonomyIndexed: valueType.IsTaxonomyFaceted); _facetFields.Add(facet); @@ -392,7 +396,8 @@ internal IFacetOperations FacetInternal(string field, params Int64Range[] longRa longRanges = Array.Empty(); } - var facet = new FacetLongField(field, longRanges, GetFacetField(field)); + var valueType = _searchContext.GetFieldValueType(field) as IIndexFacetValueType; + var facet = new FacetLongField(field, longRanges, GetFacetField(field), isTaxonomyIndexed: valueType.IsTaxonomyFaceted); _facetFields.Add(facet); @@ -407,5 +412,21 @@ private string GetFacetField(string field) } return ExamineFieldNames.DefaultFacetsName; } + private bool GetFacetFieldIsMultiValued(string field) + { + if (_facetsConfig.DimConfigs.ContainsKey(field)) + { + return _facetsConfig.DimConfigs[field].IsMultiValued; + } + return false; + } + private bool GetFacetFieldIsHierarchical(string field) + { + if (_facetsConfig.DimConfigs.ContainsKey(field)) + { + return _facetsConfig.DimConfigs[field].IsHierarchical; + } + return false; + } } } diff --git a/src/Examine.Lucene/Search/LuceneSearchResult.cs b/src/Examine.Lucene/Search/LuceneSearchResult.cs new file mode 100644 index 000000000..aa417025d --- /dev/null +++ b/src/Examine.Lucene/Search/LuceneSearchResult.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; + +namespace Examine.Lucene.Search +{ + /// + /// Lucene Index Search Result + /// + public class LuceneSearchResult : SearchResult, ISearchResult + { + /// + /// Constructor + /// + public LuceneSearchResult(string id, float score, Func>> lazyFieldVals, int shardId) + : base(id, score, lazyFieldVals) + { + ShardIndex = shardId; + } + + /// + /// Index Shard Id + /// + public int ShardIndex { get; } + } +} diff --git a/src/Examine.Lucene/Search/LuceneSearchResults.cs b/src/Examine.Lucene/Search/LuceneSearchResults.cs index 7b1131115..ab630fab1 100644 --- a/src/Examine.Lucene/Search/LuceneSearchResults.cs +++ b/src/Examine.Lucene/Search/LuceneSearchResults.cs @@ -8,26 +8,39 @@ namespace Examine.Lucene.Search /// /// Represents the search results of a query /// - public class LuceneSearchResults : ISearchResults, IFacetResults + public class LuceneSearchResults : ISearchResults, ILuceneSearchResults, IFacetResults { /// /// Gets an empty search result /// - public static LuceneSearchResults Empty { get; } = new LuceneSearchResults(Array.Empty(), 0, new Dictionary()); - + public static LuceneSearchResults Empty { get; } = new LuceneSearchResults(Array.Empty(), 0, new Dictionary(), float.NaN, default); + private readonly IReadOnlyCollection _results; /// - public LuceneSearchResults(IReadOnlyCollection results, int totalItemCount, IReadOnlyDictionary facets) + public LuceneSearchResults(IReadOnlyCollection results, int totalItemCount, IReadOnlyDictionary facets, float maxScore, SearchAfterOptions searchAfterOptions) { _results = results; TotalItemCount = totalItemCount; + MaxScore = maxScore; + SearchAfter = searchAfterOptions; Facets = facets; } /// public long TotalItemCount { get; } + /// + /// Returns the maximum score value encountered. Note that in case + /// scores are not tracked, this returns . + /// + public float MaxScore { get; } + + /// + /// Options for skipping documents after a previous search + /// + public SearchAfterOptions SearchAfter { get; } + /// public IReadOnlyDictionary Facets { get; } diff --git a/src/Examine.Lucene/Search/SearchAfterOptions.cs b/src/Examine.Lucene/Search/SearchAfterOptions.cs new file mode 100644 index 000000000..95a2312e6 --- /dev/null +++ b/src/Examine.Lucene/Search/SearchAfterOptions.cs @@ -0,0 +1,46 @@ +namespace Examine.Lucene.Search +{ + /// + /// Options for Searching After. Used for efficent deep paging. + /// + public class SearchAfterOptions + { + + /// + /// Constructor + /// + /// The Id of the last document in the previous result set. The search will search after this document + /// The Score of the last document in the previous result set. The search will search after this document + /// Search fields. Should contain null or J2N.Int + /// The index of the shard the doc belongs to + public SearchAfterOptions(int documentId, float documentScore, object[] fields, int shardIndex) + { + DocumentId = documentId; + DocumentScore = documentScore; + Fields = fields; + ShardIndex = shardIndex; + } + + /// + /// The Id of the last document in the previous result set. + /// The search will search after this document + /// + public int DocumentId { get; } + + /// + /// The Score of the last document in the previous result set. + /// The search will search after this document + /// + public float DocumentScore { get; } + + /// + /// The index of the shard the doc belongs to + /// + public int? ShardIndex { get; } + + /// + /// Search fields. Should contain null or J2N.Int + /// + public object[] Fields { get; } + } +} diff --git a/src/Examine.Lucene/Search/TaxonomySearchContext.cs b/src/Examine.Lucene/Search/TaxonomySearchContext.cs new file mode 100644 index 000000000..80d9b856a --- /dev/null +++ b/src/Examine.Lucene/Search/TaxonomySearchContext.cs @@ -0,0 +1,77 @@ +using System; +using System.Linq; +using Examine.Lucene.Indexing; +using Lucene.Net.Facet.Taxonomy; +using Lucene.Net.Index; + +namespace Examine.Lucene.Search +{ + /// + /// Taxonomy Search Context + /// + public class TaxonomySearchContext : ITaxonomySearchContext + { + private readonly SearcherTaxonomyManager _searcherManager; + private readonly FieldValueTypeCollection _fieldValueTypeCollection; + private string[] _searchableFields; + + /// + /// Constructor + /// + /// + /// + /// + public TaxonomySearchContext(SearcherTaxonomyManager searcherManager, FieldValueTypeCollection fieldValueTypeCollection) + { + _searcherManager = searcherManager ?? throw new ArgumentNullException(nameof(searcherManager)); + _fieldValueTypeCollection = fieldValueTypeCollection ?? throw new ArgumentNullException(nameof(fieldValueTypeCollection)); + } + + /// + public ISearcherReference GetSearcher() => new TaxonomySearcherReference(_searcherManager); + + /// + public string[] SearchableFields + { + get + { + if (_searchableFields == null) + { + // IMPORTANT! Do not resolve the IndexSearcher from the `IndexSearcher` property above since this + // will not release it from the searcher manager. When we are collecting fields, we are essentially + // performing a 'search'. We must ensure that the underlying reader has the correct reference counts. + var searcherAndTaxonomy = _searcherManager.Acquire(); + try + { + var fields = MultiFields.GetMergedFieldInfos(searcherAndTaxonomy.Searcher.IndexReader) + .Select(x => x.Name) + .ToList(); + + //exclude the special index fields + _searchableFields = fields + .Where(x => !x.StartsWith(ExamineFieldNames.SpecialFieldPrefix) && !x.Equals(ExamineFieldNames.DefaultFacetsName)) + .ToArray(); + } + finally + { + _searcherManager.Release(searcherAndTaxonomy); + } + } + + return _searchableFields; + } + } + + /// + public IIndexFieldValueType GetFieldValueType(string fieldName) + { + //Get the value type for the field, or use the default if not defined + return _fieldValueTypeCollection.GetValueType( + fieldName, + _fieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.FullText)); + } + + /// + public ITaxonomySearcherReference GetTaxonomyAndSearcher() => new TaxonomySearcherReference(_searcherManager); + } +} diff --git a/src/Examine.Lucene/Search/TaxonomySearcherReference.cs b/src/Examine.Lucene/Search/TaxonomySearcherReference.cs new file mode 100644 index 000000000..1afc2c2dc --- /dev/null +++ b/src/Examine.Lucene/Search/TaxonomySearcherReference.cs @@ -0,0 +1,76 @@ +using System; +using Lucene.Net.Facet.Taxonomy; +using Lucene.Net.Facet.Taxonomy.Directory; +using Lucene.Net.Search; + +namespace Examine.Lucene.Search +{ + /// + /// Represents a Taxonomy Searcher Reference + /// + public class TaxonomySearcherReference : ITaxonomySearcherReference + { + private bool _disposedValue; + private readonly SearcherTaxonomyManager _searcherManager; + private SearcherTaxonomyManager.SearcherAndTaxonomy? _searcherAndTaxonomy; + + /// + /// Constructor + /// + /// Taxonomny Searcher Manager + public TaxonomySearcherReference(SearcherTaxonomyManager searcherManager) + { + _searcherManager = searcherManager ?? throw new ArgumentNullException(nameof(searcherManager)); + } + + /// + public IndexSearcher IndexSearcher + { + get + { + if (_disposedValue) + { + throw new ObjectDisposedException($"{nameof(TaxonomySearcherReference)} is disposed"); + } + return _searcherAndTaxonomy?.Searcher ?? (_searcherAndTaxonomy = _searcherManager.Acquire()).Searcher; + } + } + + /// + public DirectoryTaxonomyReader TaxonomyReader + { + get + { + if (_disposedValue) + { + throw new ObjectDisposedException($"{nameof(TaxonomySearcherReference)} is disposed"); + } + return _searcherAndTaxonomy?.TaxonomyReader ?? (_searcherAndTaxonomy = _searcherManager.Acquire()).TaxonomyReader; + } + } + + /// + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + if (_searcherAndTaxonomy != null) + { + _searcherManager.Release(_searcherAndTaxonomy); + } + } + + _disposedValue = true; + } + } + + /// + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + } + } +} diff --git a/src/Examine.Lucene/ValueTypeFactoryCollection.cs b/src/Examine.Lucene/ValueTypeFactoryCollection.cs index d7289e59d..c325b3fb5 100644 --- a/src/Examine.Lucene/ValueTypeFactoryCollection.cs +++ b/src/Examine.Lucene/ValueTypeFactoryCollection.cs @@ -86,19 +86,31 @@ private static IReadOnlyDictionary> G {FieldDefinitionTypes.FullTextSortable, name => new FullTextType(name, loggerFactory, defaultAnalyzer, true)}, {FieldDefinitionTypes.InvariantCultureIgnoreCase, name => new GenericAnalyzerFieldValueType(name, loggerFactory, new CultureInvariantWhitespaceAnalyzer())}, {FieldDefinitionTypes.EmailAddress, name => new GenericAnalyzerFieldValueType(name, loggerFactory, new EmailAddressAnalyzer())}, - {FieldDefinitionTypes.FacetInteger, name => new Int32Type(name, loggerFactory, true, true)}, - {FieldDefinitionTypes.FacetFloat, name => new SingleType(name, loggerFactory, true, true)}, - {FieldDefinitionTypes.FacetDouble, name => new DoubleType(name, loggerFactory, true, true)}, - {FieldDefinitionTypes.FacetLong, name => new Int64Type(name, loggerFactory, true, true)}, - {FieldDefinitionTypes.FacetDateTime, name => new DateTimeType(name, loggerFactory, DateResolution.MILLISECOND, true, true)}, - {FieldDefinitionTypes.FacetDateYear, name => new DateTimeType(name, loggerFactory, DateResolution.YEAR, true, true)}, - {FieldDefinitionTypes.FacetDateMonth, name => new DateTimeType(name, loggerFactory, DateResolution.MONTH, true, true)}, - {FieldDefinitionTypes.FacetDateDay, name => new DateTimeType(name, loggerFactory, DateResolution.DAY, true, true)}, - {FieldDefinitionTypes.FacetDateHour, name => new DateTimeType(name, loggerFactory, DateResolution.HOUR, true, true)}, - {FieldDefinitionTypes.FacetDateMinute, name => new DateTimeType(name, loggerFactory, DateResolution.MINUTE, true, true)}, - {FieldDefinitionTypes.FacetFullText, name => new FullTextType(name, loggerFactory, false, true, defaultAnalyzer)}, + {FieldDefinitionTypes.FacetInteger, name => new Int32Type(name, loggerFactory, true, true, false)}, + {FieldDefinitionTypes.FacetFloat, name => new SingleType(name, loggerFactory, true, true, false)}, + {FieldDefinitionTypes.FacetDouble, name => new DoubleType(name, loggerFactory, true, true, false)}, + {FieldDefinitionTypes.FacetLong, name => new Int64Type(name, loggerFactory, true, true, false)}, + {FieldDefinitionTypes.FacetDateTime, name => new DateTimeType(name, loggerFactory, DateResolution.MILLISECOND, true, true, false)}, + {FieldDefinitionTypes.FacetDateYear, name => new DateTimeType(name, loggerFactory, DateResolution.YEAR, true, true, false)}, + {FieldDefinitionTypes.FacetDateMonth, name => new DateTimeType(name, loggerFactory, DateResolution.MONTH, true, true, false)}, + {FieldDefinitionTypes.FacetDateDay, name => new DateTimeType(name, loggerFactory, DateResolution.DAY, true, true, false)}, + {FieldDefinitionTypes.FacetDateHour, name => new DateTimeType(name, loggerFactory, DateResolution.HOUR, true, true, false)}, + {FieldDefinitionTypes.FacetDateMinute, name => new DateTimeType(name, loggerFactory, DateResolution.MINUTE, true, true, false)}, + {FieldDefinitionTypes.FacetFullText, name => new FullTextType(name, loggerFactory, true, true, defaultAnalyzer)}, {FieldDefinitionTypes.FacetFullTextSortable, name => new FullTextType(name, loggerFactory, true, true, defaultAnalyzer)}, - }; + {FieldDefinitionTypes.FacetTaxonomyInteger, name => new Int32Type(name, loggerFactory, true,isFacetable: true, taxonomyIndex: true)}, + {FieldDefinitionTypes.FacetTaxonomyFloat, name => new SingleType(name, loggerFactory, true,isFacetable: true, taxonomyIndex: true)}, + {FieldDefinitionTypes.FacetTaxonomyDouble, name => new DoubleType(name, loggerFactory, true,isFacetable: true, taxonomyIndex: true)}, + {FieldDefinitionTypes.FacetTaxonomyLong, name => new Int64Type(name, loggerFactory, true, isFacetable: true, taxonomyIndex: true)}, + {FieldDefinitionTypes.FacetTaxonomyDateTime, name => new DateTimeType(name, loggerFactory, DateResolution.MILLISECOND, true, isFacetable: true, taxonomyIndex: true)}, + {FieldDefinitionTypes.FacetTaxonomyDateYear, name => new DateTimeType(name, loggerFactory, DateResolution.YEAR, true,isFacetable: true, taxonomyIndex: true)}, + {FieldDefinitionTypes.FacetTaxonomyDateMonth, name =>new DateTimeType(name, loggerFactory, DateResolution.MONTH, true,isFacetable: true, taxonomyIndex: true)}, + {FieldDefinitionTypes.FacetTaxonomyDateDay, name =>new DateTimeType(name, loggerFactory, DateResolution.DAY, true,isFacetable: true, taxonomyIndex: true)}, + {FieldDefinitionTypes.FacetTaxonomyDateHour, name => new DateTimeType(name, loggerFactory, DateResolution.HOUR, true,isFacetable: true, taxonomyIndex: true)}, + {FieldDefinitionTypes.FacetTaxonomyDateMinute, name => new DateTimeType(name, loggerFactory, DateResolution.MINUTE, true, isFacetable: true, taxonomyIndex: true)}, + {FieldDefinitionTypes.FacetTaxonomyFullText, name => new FullTextType(name, loggerFactory, false, true, defaultAnalyzer, taxonomyIndex: true)}, + {FieldDefinitionTypes.FacetTaxonomyFullTextSortable, name => new FullTextType(name, loggerFactory, true, true, defaultAnalyzer, taxonomyIndex: true)}, + }; /// diff --git a/src/Examine.Test/Examine.Lucene/ExamineTaxonomyReplicatorTests.cs b/src/Examine.Test/Examine.Lucene/ExamineTaxonomyReplicatorTests.cs new file mode 100644 index 000000000..b840e94ab --- /dev/null +++ b/src/Examine.Test/Examine.Lucene/ExamineTaxonomyReplicatorTests.cs @@ -0,0 +1,186 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using Examine.Lucene; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Index; +using Microsoft.Extensions.Logging; +using NUnit.Framework; + +namespace Examine.Test.Examine.Lucene.Sync +{ + [TestFixture] + public class ExamineTaxonomyReplicatorTests : ExamineBaseTest + { + private ILoggerFactory GetLoggerFactory() + => LoggerFactory.Create(x => x.AddConsole().SetMinimumLevel(LogLevel.Debug)); + + [Test] + public void GivenAMainIndex_WhenReplicatedLocally_TheLocalIndexIsPopulated() + { + var tempStorage = new System.IO.DirectoryInfo(TestContext.CurrentContext.WorkDirectory); + var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); + + using (var mainDir = new RandomIdRAMDirectory()) + using (var localDir = new RandomIdRAMDirectory()) + using (var mainTaxonomyDir = new RandomIdRAMDirectory()) + using (var localTaxonomyDir = new RandomIdRAMDirectory()) + using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + using (var replicator = new ExamineTaxonomyReplicator(GetLoggerFactory(), mainIndex, localDir, localTaxonomyDir, tempStorage)) + { + mainIndex.CreateIndex(); + + mainIndex.IndexItems(mainIndex.AllData()); + + DirectoryReader mainReader = mainIndex.IndexWriter.IndexWriter.GetReader(true); + Assert.AreEqual(100, mainReader.NumDocs); + + // TODO: Ok so replication CANNOT occur on an open index with an open IndexWriter. + // See this note: https://lucenenet.apache.org/docs/4.8.0-beta00014/api/replicator/Lucene.Net.Replicator.IndexReplicationHandler.html + // "NOTE: This handler assumes that Lucene.Net.Index.IndexWriter is not opened by another process on the index directory. In fact, opening an Lucene.Net.Index.IndexWriter on the same directory to which files are copied can lead to undefined behavior, where some or all the files will be deleted, override other files or simply create a mess. When you replicate an index, it is best if the index is never modified by Lucene.Net.Index.IndexWriter, except the one that is open on the source index, from which you replicate." + // So if we want to replicate, we can sync from Main on startup and ensure that the writer isn't opened until that + // is done (the callback can be used for that). + // If we want to sync back to main, it means we can never open a writer to main, but that might be ok and we + // publish on a schedule. + replicator.ReplicateIndex(); + + using (TestIndex localIndex = GetTaxonomyTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + { + DirectoryReader localReader = localIndex.IndexWriter.IndexWriter.GetReader(true); + Assert.AreEqual(100, localReader.NumDocs); + } + } + } + + [Test] + public void GivenAnOpenedWriter_WhenReplicationAttempted_ThenAnExceptionIsThrown() + { + var tempStorage = new System.IO.DirectoryInfo(TestContext.CurrentContext.WorkDirectory); + var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); + + using (var mainDir = new RandomIdRAMDirectory()) + using (var localDir = new RandomIdRAMDirectory()) + using (var mainTaxonomyDir = new RandomIdRAMDirectory()) + using (var localTaxonomyDir = new RandomIdRAMDirectory()) + using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + using (var replicator = new ExamineTaxonomyReplicator(GetLoggerFactory(), mainIndex, localDir, localTaxonomyDir, tempStorage)) + using (TestIndex localIndex = GetTaxonomyTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + { + mainIndex.CreateIndex(); + + // this will open the writer + localIndex.IndexItem(new ValueSet(9999.ToString(), "content", + new Dictionary> + { + {"item1", new List(new[] {"value1"})}, + {"item2", new List(new[] {"value2"})} + })); + + mainIndex.IndexItems(mainIndex.AllData()); + + Assert.Throws(() => replicator.ReplicateIndex()); + } + } + + [Test] + public void GivenASyncedLocalIndex_WhenTriggered_ThenSyncedBackToMainIndex() + { + var tempStorage = new System.IO.DirectoryInfo(TestContext.CurrentContext.WorkDirectory); + var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); + + using (var mainDir = new RandomIdRAMDirectory()) + using (var localDir = new RandomIdRAMDirectory()) + using (var mainTaxonomyDir = new RandomIdRAMDirectory()) + using (var localTaxonomyDir = new RandomIdRAMDirectory()) + { + using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + using (var replicator = new ExamineTaxonomyReplicator(GetLoggerFactory(), mainIndex, localDir, localTaxonomyDir, tempStorage)) + { + mainIndex.CreateIndex(); + mainIndex.IndexItems(mainIndex.AllData()); + replicator.ReplicateIndex(); + } + + using (TestIndex localIndex = GetTaxonomyTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + { + localIndex.IndexItem(new ValueSet(9999.ToString(), "content", + new Dictionary> + { + {"item1", new List(new[] {"value1"})}, + {"item2", new List(new[] {"value2"})} + })); + + using (var replicator = new ExamineTaxonomyReplicator(GetLoggerFactory(), localIndex, mainDir, mainTaxonomyDir, tempStorage)) + { + // replicate back to main, main index must be closed + replicator.ReplicateIndex(); + } + + using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + { + DirectoryReader mainReader = mainIndex.IndexWriter.IndexWriter.GetReader(true); + Assert.AreEqual(101, mainReader.NumDocs); + } + } + } + + } + + [Test] + public void GivenASyncedLocalIndex_ThenSyncedBackToMainIndexOnSchedule() + { + var tempStorage = new System.IO.DirectoryInfo(TestContext.CurrentContext.WorkDirectory); + var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); + + using (var mainDir = new RandomIdRAMDirectory()) + using (var localDir = new RandomIdRAMDirectory()) + using (var mainTaxonomyDir = new RandomIdRAMDirectory()) + using (var localTaxonomyDir = new RandomIdRAMDirectory()) + { + using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + using (var replicator = new ExamineTaxonomyReplicator(GetLoggerFactory(), mainIndex, localDir, localTaxonomyDir, tempStorage)) + { + mainIndex.CreateIndex(); + mainIndex.IndexItems(mainIndex.AllData()); + replicator.ReplicateIndex(); + } + + using (TestIndex localIndex = GetTaxonomyTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + { + using (var replicator = new ExamineTaxonomyReplicator( + GetLoggerFactory(), + localIndex, + mainDir, + mainTaxonomyDir, + tempStorage)) + { + // replicate back to main on schedule + replicator.StartIndexReplicationOnSchedule(1000); + + for (int i = 0; i < 10; i++) + { + localIndex.IndexItem(new ValueSet(("testing" + i).ToString(), "content", + new Dictionary> + { + {"item1", new List(new[] {"value1"})}, + {"item2", new List(new[] {"value2"})} + })); + + Thread.Sleep(500); + } + + // should be plenty to resync everything + Thread.Sleep(2000); + } + + using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + { + DirectoryReader mainReader = mainIndex.IndexWriter.IndexWriter.GetReader(true); + Assert.AreEqual(110, mainReader.NumDocs); + } + } + } + + } + } +} diff --git a/src/Examine.Test/Examine.Lucene/Search/FluentApiTests.cs b/src/Examine.Test/Examine.Lucene/Search/FluentApiTests.cs index 2d0bee137..a054531d9 100644 --- a/src/Examine.Test/Examine.Lucene/Search/FluentApiTests.cs +++ b/src/Examine.Test/Examine.Lucene/Search/FluentApiTests.cs @@ -5,7 +5,6 @@ using Examine.Lucene.Providers; using Examine.Lucene.Search; using Examine.Search; -using J2N; using Lucene.Net.Analysis.En; using Lucene.Net.Analysis.Standard; using Lucene.Net.Facet; @@ -21,16 +20,40 @@ namespace Examine.Test.Examine.Lucene.Search [Parallelizable(ParallelScope.All)] public class FluentApiTests : ExamineBaseTest { - [TestCase(true)] - [TestCase(false)] - public void Allow_Leading_Wildcards(bool withFacets) + public enum FacetTestType { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + NoFacets, + TaxonomyFacets, + SortedSetFacets + } + + private bool HasFacets(FacetTestType withFacets) => withFacets == FacetTestType.TaxonomyFacets || withFacets == FacetTestType.SortedSetFacets; + + + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Allow_Leading_Wildcards(FacetTestType withFacets) + { + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -62,7 +85,7 @@ public void Allow_Leading_Wildcards(bool withFacets) AllowLeadingWildcard = false }).NativeQuery("*dney")); - if (withFacets) + if (HasFacets(withFacets)) { var results1 = query1.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -77,21 +100,35 @@ public void Allow_Leading_Wildcards(bool withFacets) var results1 = query1.Execute(); Assert.AreEqual(2, results1.TotalItemCount); - } + } } } - [TestCase(true)] - [TestCase(false)] - public void NativeQuery_Single_Word(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void NativeQuery_Single_Word(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -110,7 +147,7 @@ public void NativeQuery_Single_Word(bool withFacets) Console.WriteLine(query); - if (withFacets) + if (HasFacets(withFacets)) { var results = query.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -129,18 +166,33 @@ public void NativeQuery_Single_Word(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Uppercase_Category(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Uppercase_Category(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, + //Ensure it's set to a fulltextsortable, otherwise it's not sortable fieldDefinitionCollection)) { indexer.IndexItems(new[] { @@ -158,7 +210,7 @@ public void Uppercase_Category(bool withFacets) Console.WriteLine(query); - if (withFacets) + if (HasFacets(withFacets)) { var results = query.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -190,7 +242,8 @@ public void FacetsConfig_SetIndexName_FullText() luceneDir, analyzer, fieldDefinitionCollection, - facetsConfig: facetsConfig)) + facetsConfig: + facetsConfig)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "cOntent", @@ -200,14 +253,11 @@ public void FacetsConfig_SetIndexName_FullText() ValueSet.FromObject(3.ToString(), "cOntent", new { nodeName = "location 3", bodyText = "Sydney is the capital of NSW in Australia"}) }); - var searcher = indexer.Searcher; - var query = searcher.CreateQuery("cOntent").All(); Console.WriteLine(query); - var results = query.WithFacets(facets => facets.Facet("nodeName")).Execute(); var facetResults = results.GetFacet("nodeName"); @@ -277,6 +327,148 @@ public void FacetsConfig_SetIndexName_Double() luceneDir, analyzer, fieldDefinitionCollection, + facetsConfig: + facetsConfig)) + { + indexer.IndexItems(new[] { + ValueSet.FromObject(1.ToString(), "cOntent", + new { nodeName = "location 1", bodyText = "Zanzibar is in Africa", DoubleValue = 10D }), + ValueSet.FromObject(2.ToString(), "cOntent", + new { nodeName = "location 2", bodyText = "In Canada there is a town called Sydney in Nova Scotia", DoubleValue = 20D }), + ValueSet.FromObject(3.ToString(), "cOntent", + new { nodeName = "location 3", bodyText = "Sydney is the capital of NSW in Australia", DoubleValue = 30D }) + }); + + var searcher = indexer.Searcher; + + var query = searcher.CreateQuery("cOntent").All(); + + Console.WriteLine(query); + + + var results = query.WithFacets(factes => factes.Facet("DoubleValue", new DoubleRange[] + { + new DoubleRange("10", 10, true, 11, true), + new DoubleRange("20", 20, true, 21, true), + new DoubleRange("30", 30, true, 31, true), + })).Execute(); + + var facetResults = results.GetFacet("DoubleValue"); + + Assert.AreEqual(3, results.TotalItemCount); + Assert.AreEqual(3, facetResults.Count()); + } + } + + [Test] + public void Taxonomy_FacetsConfig_SetIndexName_FullText() + { + var fieldDefinitionCollection = new FieldDefinitionCollection( + new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + + var facetsConfig = new FacetsConfig(); + facetsConfig.SetIndexFieldName("nodeName", "facet_nodeName"); + + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection, + facetsConfig: facetsConfig)) + { + indexer.IndexItems(new[] { + ValueSet.FromObject(1.ToString(), "cOntent", + new { nodeName = "location 1", bodyText = "Zanzibar is in Africa"}), + ValueSet.FromObject(2.ToString(), "cOntent", + new { nodeName = "location 2", bodyText = "In Canada there is a town called Sydney in Nova Scotia"}), + ValueSet.FromObject(3.ToString(), "cOntent", + new { nodeName = "location 3", bodyText = "Sydney is the capital of NSW in Australia"}) + }); + + var searcher = indexer.Searcher; + + var query = searcher.CreateQuery("cOntent").All(); + + Console.WriteLine(query); + + + var results = query.WithFacets(facets => facets.Facet("nodeName")).Execute(); + + var facetResults = results.GetFacet("nodeName"); + + Assert.AreEqual(3, results.TotalItemCount); + Assert.AreEqual(3, facetResults.Count()); + } + } + + [Test] + public void Taxonomy_FacetsConfig_SetIndexName_Long() + { + var fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("LongValue", FieldDefinitionTypes.FacetTaxonomyLong)); + + var facetsConfig = new FacetsConfig(); + facetsConfig.SetIndexFieldName("LongValue", "facet_longvalue"); + + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection, + facetsConfig: facetsConfig)) + { + indexer.IndexItems(new[] { + ValueSet.FromObject(1.ToString(), "cOntent", + new { nodeName = "location 1", bodyText = "Zanzibar is in Africa", LongValue = 10L }), + ValueSet.FromObject(2.ToString(), "cOntent", + new { nodeName = "location 2", bodyText = "In Canada there is a town called Sydney in Nova Scotia", LongValue = 20L }), + ValueSet.FromObject(3.ToString(), "cOntent", + new { nodeName = "location 3", bodyText = "Sydney is the capital of NSW in Australia", LongValue = 30L }) + }); + + var searcher = indexer.Searcher; + + var query = searcher.CreateQuery("cOntent").All(); + + Console.WriteLine(query); + + + var results = query.WithFacets(facets => facets.Facet("LongValue", new Int64Range[] + { + new Int64Range("10", 10, true, 11, true), + new Int64Range("20", 20, true, 21, true), + new Int64Range("30", 30, true, 31, true), + })).Execute(); + + var facetResults = results.GetFacet("LongValue"); + + Assert.AreEqual(3, results.TotalItemCount); + Assert.AreEqual(3, facetResults.Count()); + } + } + + [Test] + public void Taxonomy_FacetsConfig_SetIndexName_Double() + { + var fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), + new FieldDefinition("DoubleValue", FieldDefinitionTypes.FacetTaxonomyDouble)); + + var facetsConfig = new FacetsConfig(); + facetsConfig.SetIndexFieldName("DoubleValue", "facet_doublevalue"); + + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection, facetsConfig: facetsConfig)) { indexer.IndexItems(new[] { @@ -309,17 +501,31 @@ public void FacetsConfig_SetIndexName_Double() } } - [TestCase(true)] - [TestCase(false)] - public void NativeQuery_Phrase(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void NativeQuery_Phrase(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("bodyText", FieldDefinitionTypes.FacetFullText)) - : new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("bodyText", FieldDefinitionTypes.FacetTaxonomyFullText), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("bodyText", FieldDefinitionTypes.FacetFullText), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("bodyText", FieldDefinitionTypes.FullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -339,7 +545,7 @@ public void NativeQuery_Phrase(bool withFacets) Console.WriteLine(query); Assert.AreEqual("{ Category: content, LuceneQuery: +(nodeName:\"town called\" bodyText:\"town called\") }", query.ToString()); - if (withFacets) + if (HasFacets(withFacets)) { var results = query.WithFacets(facets => facets.Facet("bodyText")).Execute(); @@ -357,13 +563,25 @@ public void NativeQuery_Phrase(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Managed_Range_Date(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Managed_Range_Date(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("created", "datetime"), new FieldDefinition("created", FieldDefinitionTypes.FacetDateTime)) - : new FieldDefinitionCollection(new FieldDefinition("created", "datetime")); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("created", "datetime"), new FieldDefinition("created", FieldDefinitionTypes.FacetDateTime)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("created", "datetime"), new FieldDefinition("created", FieldDefinitionTypes.FacetTaxonomyDateTime)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("created", "datetime")); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var indexer = GetTestIndex( @@ -404,7 +622,7 @@ public void Managed_Range_Date(bool withFacets) var numberSortedCriteria = searcher.CreateQuery() .RangeQuery(new[] { "created" }, new DateTime(2000, 01, 02), new DateTime(2000, 01, 05), maxInclusive: false); - if(withFacets) + if (HasFacets(withFacets)) { var numberSortedResult = numberSortedCriteria.WithFacets(facets => facets.Facet("created", new Int64Range[] { @@ -430,17 +648,31 @@ public void Managed_Range_Date(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Managed_Full_Text(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Managed_Full_Text(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("item1", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("item1", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("item1", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir1 = new RandomIdRAMDirectory()) - using (var indexer1 = GetTestIndex(luceneDir1, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer1 = GetTaxonomyTestIndex( + luceneDir1, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer1.IndexItem(ValueSet.FromObject("1", "content", new { item1 = "value1", item2 = "The agitated zebras gallop back and forth in short, panicky dashes, then skitter off into the total absolute darkness." })); indexer1.IndexItem(ValueSet.FromObject("2", "content", new { item1 = "value2", item2 = "The festival lasts five days and celebrates the victory of good over evil, light over darkness, and knowledge over ignorance." })); @@ -452,7 +684,7 @@ public void Managed_Full_Text(bool withFacets) var searcher = indexer1.Searcher; - if (withFacets) + if (HasFacets(withFacets)) { var result = searcher.CreateQuery() .ManagedQuery("darkness") @@ -506,17 +738,31 @@ public void Managed_Full_Text(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Managed_Full_Text_With_Bool(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Managed_Full_Text_With_Bool(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("item1", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("item1", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("item1", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir1 = new RandomIdRAMDirectory()) - using (var indexer1 = GetTestIndex(luceneDir1, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer1 = GetTaxonomyTestIndex( + luceneDir1, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer1.IndexItem(ValueSet.FromObject("1", "content", new { item1 = "value1", item2 = "The agitated zebras gallop back and forth in short, panicky dashes, then skitter off into the total absolute darkness." })); indexer1.IndexItem(ValueSet.FromObject("2", "content", new { item1 = "value2", item2 = "The festival lasts five days and celebrates the victory of good over evil, light over darkness, and knowledge over ignorance." })); @@ -530,7 +776,7 @@ public void Managed_Full_Text_With_Bool(bool withFacets) var qry = searcher.CreateQuery().ManagedQuery("darkness").And().Field("item1", "value1"); Console.WriteLine(qry); - if (withFacets) + if (HasFacets(withFacets)) { var result = qry.WithFacets(facets => facets.Facet("item1")).Execute(); @@ -585,17 +831,31 @@ public void Managed_Full_Text_With_Bool(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Not_Managed_Full_Text(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Not_Managed_Full_Text(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("item1", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("item1", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("item1", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir1 = new RandomIdRAMDirectory()) - using (var indexer1 = GetTestIndex(luceneDir1, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer1 = GetTaxonomyTestIndex( + luceneDir1, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer1.IndexItem(ValueSet.FromObject("1", "content", new { item1 = "value1", item2 = "The agitated zebras gallop back and forth in short, panicky dashes, then skitter off into the total absolute chaos." })); indexer1.IndexItem(ValueSet.FromObject("2", "content", new { item1 = "value1", item2 = "The festival lasts five days and celebrates the victory of good over evil, light over darkness, and knowledge over ignorance." })); @@ -612,7 +872,7 @@ public void Not_Managed_Full_Text(bool withFacets) Console.WriteLine(qry); - if (withFacets) + if (HasFacets(withFacets)) { var result = qry.WithFacets(facets => facets.Facet("item1")).Execute(); @@ -644,17 +904,31 @@ public void Not_Managed_Full_Text(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Managed_Range_Int(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Managed_Range_Int(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.FacetInteger)) - : new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.FacetTaxonomyInteger)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.FacetInteger)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -690,7 +964,7 @@ public void Managed_Range_Int(bool withFacets) var numberSortedCriteria = searcher.CreateQuery() .RangeQuery(new[] { "parentID" }, 122, 124); - if (withFacets) + if (HasFacets(withFacets)) { var numberSortedResult = numberSortedCriteria .WithFacets(facets => facets.Facet("parentID", new Int64Range[] @@ -716,17 +990,31 @@ public void Managed_Range_Int(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Legacy_ParentId(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Legacy_ParentId(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.FacetInteger)) - : new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.FacetTaxonomyInteger)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.FacetInteger)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -763,7 +1051,7 @@ public void Legacy_ParentId(bool withFacets) .Field("parentID", 123) .OrderBy(new SortableField("sortOrder", SortType.Int)); - if (withFacets) + if (HasFacets(withFacets)) { var numberSortedResult = numberSortedCriteria.WithFacets(facets => facets.Facet("parentID")).Execute(); @@ -784,16 +1072,30 @@ public void Legacy_ParentId(bool withFacets) } - [TestCase(true)] - [TestCase(false)] - public void Grouped_Or_Examiness(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Grouped_Or_Examiness(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { @@ -832,7 +1134,7 @@ public void Grouped_Or_Examiness(bool withFacets) Console.WriteLine(filter); - if (withFacets) + if (HasFacets(withFacets)) { var results = filter.WithFacets(facets => facets.Facet("nodeTypeAlias")).Execute(); @@ -1014,16 +1316,30 @@ public void Grouped_Not_Query_Output() } } - [TestCase(true)] - [TestCase(false)] - public void Grouped_Not_Single_Field_Single_Value(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Grouped_Not_Single_Field_Single_Value(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { @@ -1039,7 +1355,7 @@ public void Grouped_Not_Single_Field_Single_Value(bool withFacets) query.GroupedNot(new[] { "umbracoNaviHide" }, 1.ToString()); Console.WriteLine(query.Query); - if (withFacets) + if (HasFacets(withFacets)) { var results = query.All().WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -1056,16 +1372,29 @@ public void Grouped_Not_Single_Field_Single_Value(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Grouped_Not_Multi_Field_Single_Value(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Grouped_Not_Multi_Field_Single_Value(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { @@ -1084,7 +1413,7 @@ public void Grouped_Not_Multi_Field_Single_Value(bool withFacets) var query = searcher.CreateQuery("content").GroupedNot(new[] { "umbracoNaviHide", "show" }, 1.ToString()); Console.WriteLine(query); - if (withFacets) + if (HasFacets(withFacets)) { var results = query.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -1102,22 +1431,33 @@ public void Grouped_Not_Multi_Field_Single_Value(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Grouped_Or_With_Not(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Grouped_Or_With_Not(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("headerText", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("headerText", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("headerText", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( - + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, //TODO: Making this a number makes the query fail - i wonder how to make it work correctly? // It's because the searching is NOT using a managed search //new[] { new FieldDefinition("umbracoNaviHide", FieldDefinitionTypes.Integer) }, - - luceneDir, analyzer, fieldDefinitionCollection)) + fieldDefinitionCollection)) { @@ -1134,7 +1474,7 @@ public void Grouped_Or_With_Not(bool withFacets) var criteria = searcher.CreateQuery("content"); var filter = criteria.GroupedOr(new[] { "nodeName", "bodyText", "headerText" }, "ipsum").Not().Field("umbracoNaviHide", "1"); - if (withFacets) + if (HasFacets(withFacets)) { var results = filter.WithFacets(facets => facets.Facet("headerText")).Execute(); @@ -1151,16 +1491,30 @@ public void Grouped_Or_With_Not(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void And_Grouped_Not_Single_Value(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void And_Grouped_Not_Single_Value(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -1178,7 +1532,7 @@ public void And_Grouped_Not_Single_Value(bool withFacets) Console.WriteLine(query); - if (withFacets) + if (HasFacets(withFacets)) { var results = query.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -1195,16 +1549,30 @@ public void And_Grouped_Not_Single_Value(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void And_Grouped_Not_Multi_Value(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void And_Grouped_Not_Multi_Value(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -1222,7 +1590,7 @@ public void And_Grouped_Not_Multi_Value(bool withFacets) Console.WriteLine(query); - if (withFacets) + if (HasFacets(withFacets)) { var results = query.WithFacets(facets => facets.Facet("nodeName")).Execute(); var facetResults = results.GetFacet("nodeName"); @@ -1238,16 +1606,30 @@ public void And_Grouped_Not_Multi_Value(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void And_Not_Single_Field(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void And_Not_Single_Field(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -1265,7 +1647,7 @@ public void And_Not_Single_Field(bool withFacets) Console.WriteLine(query); - if (withFacets) + if (HasFacets(withFacets)) { var results = query.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -1283,16 +1665,30 @@ public void And_Not_Single_Field(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void AndNot_Nested(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void AndNot_Nested(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -1313,7 +1709,7 @@ public void AndNot_Nested(bool withFacets) Console.WriteLine(query); - if (withFacets) + if (HasFacets(withFacets)) { var results = query.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -1329,16 +1725,30 @@ public void AndNot_Nested(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void And_Not_Added_Later(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void And_Not_Added_Later(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -1359,7 +1769,7 @@ public void And_Not_Added_Later(bool withFacets) Console.WriteLine(query); - if (withFacets) + if (HasFacets(withFacets)) { var results = query.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -1375,16 +1785,33 @@ public void And_Not_Added_Later(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Not_Range(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Not_Range(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("start", FieldDefinitionTypes.FacetInteger)) - : new FieldDefinitionCollection(new FieldDefinition("start", FieldDefinitionTypes.Integer)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("start", FieldDefinitionTypes.FacetTaxonomyInteger)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("start", FieldDefinitionTypes.FacetInteger)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("start", FieldDefinitionTypes.Integer)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -1401,7 +1828,7 @@ public void Not_Range(bool withFacets) Console.WriteLine(query); - if (withFacets) + if (HasFacets(withFacets)) { var results = query @@ -1422,18 +1849,32 @@ public void Not_Range(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Match_By_Path(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Match_By_Path(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("__Path", "raw"), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : new FieldDefinitionCollection(new FieldDefinition("__Path", "raw")); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("__Path", "raw"), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("__Path", "raw"), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("__Path", "raw")); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -1464,7 +1905,7 @@ public void Match_By_Path(bool withFacets) var criteria = searcher.CreateQuery("content"); var filter = criteria.Field("__Path", "-1,123,456,789"); - if (withFacets) + if (HasFacets(withFacets)) { var results1 = filter.WithFacets(facets => facets.Facet("nodeName")).Execute(); var facetResults1 = results1.GetFacet("nodeName"); @@ -1481,7 +1922,7 @@ public void Match_By_Path(bool withFacets) var exactcriteria = searcher.CreateQuery("content"); var exactfilter = exactcriteria.Field("__Path", "-1,123,456,789".Escape()); - if (withFacets) + if (HasFacets(withFacets)) { var results2 = exactfilter.WithFacets(facets => facets.Facet("nodeName")).Execute(); var facetResults2 = results2.GetFacet("nodeName"); @@ -1499,7 +1940,7 @@ public void Match_By_Path(bool withFacets) var nativeFilter = nativeCriteria.NativeQuery("__Path:\\-1,123,456,789"); Console.WriteLine(nativeFilter); - if (withFacets) + if (HasFacets(withFacets)) { var results5 = nativeFilter.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -1517,7 +1958,7 @@ public void Match_By_Path(bool withFacets) var wildcardcriteria = searcher.CreateQuery("content"); var wildcardfilter = wildcardcriteria.Field("__Path", "-1,123,456,".MultipleCharacterWildcard()); - if (withFacets) + if (HasFacets(withFacets)) { var results3 = wildcardfilter.WithFacets(facets => facets.Facet("nodeName")).Execute(); var facetResults3 = results3.GetFacet("nodeName"); @@ -1534,7 +1975,7 @@ public void Match_By_Path(bool withFacets) wildcardcriteria = searcher.CreateQuery("content"); wildcardfilter = wildcardcriteria.Field("__Path", "-1,123,457,".MultipleCharacterWildcard()); - if (withFacets) + if (HasFacets(withFacets)) { var results3 = wildcardfilter.WithFacets(facets => facets.Facet("nodeName")).Execute(); var facetResults3 = results3.GetFacet("nodeName"); @@ -1551,17 +1992,31 @@ public void Match_By_Path(bool withFacets) } - [TestCase(true)] - [TestCase(false)] - public void Find_By_ParentId(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Find_By_ParentId(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -1579,7 +2034,7 @@ public void Find_By_ParentId(bool withFacets) var criteria = searcher.CreateQuery("content"); var filter = criteria.Field("parentID", 1139); - if (withFacets) + if (HasFacets(withFacets)) { var results = filter.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -1597,17 +2052,31 @@ public void Find_By_ParentId(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Find_By_ParentId_Native_Query(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Find_By_ParentId_Native_Query(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.FacetInteger)) - : new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.FacetInteger)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.FacetInteger)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -1636,7 +2105,7 @@ public void Find_By_ParentId_Native_Query(bool withFacets) //We can use a Lucene query directly instead: //((LuceneSearchQuery)criteria).LuceneQuery(NumericRangeQuery) - if (withFacets) + if (HasFacets(withFacets)) { var results = filter.WithFacets(facets => facets.Facet("parentID")).Execute(); @@ -1655,17 +2124,31 @@ public void Find_By_ParentId_Native_Query(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Find_By_NodeTypeAlias(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Find_By_NodeTypeAlias(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", "raw"), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", "raw")); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", "raw"), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", "raw"), new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", "raw")); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -1702,7 +2185,7 @@ public void Find_By_NodeTypeAlias(bool withFacets) var criteria = searcher.CreateQuery("content"); var filter = criteria.Field("nodeTypeAlias", "CWS_Home".Escape()); - if (withFacets) + if (HasFacets(withFacets)) { var results = filter.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -1720,18 +2203,30 @@ public void Find_By_NodeTypeAlias(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Search_With_Stop_Words(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Search_With_Stop_Words(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) - - + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -1754,7 +2249,7 @@ public void Search_With_Stop_Words(bool withFacets) Console.WriteLine(filter); - if (withFacets) + if (HasFacets(withFacets)) { var results = filter.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -1772,18 +2267,30 @@ public void Search_With_Stop_Words(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Search_Native_Query(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Search_Native_Query(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) - - + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { @@ -1815,7 +2322,7 @@ public void Search_Native_Query(bool withFacets) var criteria = searcher.CreateQuery("content").NativeQuery("nodeTypeAlias:CWS_Home"); - if (withFacets) + if (HasFacets(withFacets)) { var results = criteria.WithFacets(facets => facets.Facet("nodeTypeAlias")).Execute(); @@ -1836,18 +2343,30 @@ public void Search_Native_Query(bool withFacets) } - [TestCase(true)] - [TestCase(false)] - public void Find_Only_Image_Media(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Find_Only_Image_Media(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeTypeAlias", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) - - + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { @@ -1865,7 +2384,7 @@ public void Find_Only_Image_Media(bool withFacets) var criteria = searcher.CreateQuery("media"); var filter = criteria.Field("nodeTypeAlias", "image"); - if (withFacets) + if (HasFacets(withFacets)) { var results = filter.WithFacets(facets => facets.Facet("nodeTypeAlias")).Execute(); @@ -1884,16 +2403,29 @@ public void Find_Only_Image_Media(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Find_Both_Media_And_Content(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Find_Both_Media_And_Content(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "media", @@ -1914,7 +2446,7 @@ public void Find_Both_Media_And_Content(bool withFacets) .Or() .Field(ExamineFieldNames.CategoryFieldName, "content"); - if (withFacets) + if (HasFacets(withFacets)) { var results = filter.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -1932,17 +2464,31 @@ public void Find_Both_Media_And_Content(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Sort_Result_By_Number_Field(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Sort_Result_By_Number_Field(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("sortOrder", FieldDefinitionTypes.FacetInteger), new FieldDefinition("parentID", FieldDefinitionTypes.FacetInteger)) - : new FieldDefinitionCollection(new FieldDefinition("sortOrder", FieldDefinitionTypes.Integer), new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("sortOrder", FieldDefinitionTypes.FacetTaxonomyInteger), new FieldDefinition("parentID", FieldDefinitionTypes.FacetTaxonomyInteger)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("sortOrder", FieldDefinitionTypes.FacetInteger), new FieldDefinition("parentID", FieldDefinitionTypes.FacetInteger)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("sortOrder", FieldDefinitionTypes.Integer), new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, //Ensure it's set to a number, otherwise it's not sortable fieldDefinitionCollection)) @@ -1963,7 +2509,7 @@ public void Sort_Result_By_Number_Field(bool withFacets) var sc = searcher.CreateQuery("content"); var sc1 = sc.Field("parentID", 1143).OrderBy(new SortableField("sortOrder", SortType.Int)); - if (withFacets) + if (HasFacets(withFacets)) { var results1 = sc1 .WithFacets(facets => facets @@ -2002,17 +2548,31 @@ public void Sort_Result_By_Number_Field(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Sort_Result_By_Date_Field(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Sort_Result_By_Date_Field(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("updateDate", FieldDefinitionTypes.FacetDateTime), new FieldDefinition("parentID", FieldDefinitionTypes.FacetInteger)) - : new FieldDefinitionCollection(new FieldDefinition("updateDate", FieldDefinitionTypes.DateTime), new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("updateDate", FieldDefinitionTypes.FacetTaxonomyDateTime), new FieldDefinition("parentID", FieldDefinitionTypes.FacetTaxonomyInteger)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("updateDate", FieldDefinitionTypes.FacetDateTime), new FieldDefinition("parentID", FieldDefinitionTypes.FacetInteger)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("updateDate", FieldDefinitionTypes.DateTime), new FieldDefinition("parentID", FieldDefinitionTypes.Integer)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, //Ensure it's set to a date, otherwise it's not sortable fieldDefinitionCollection)) @@ -2038,7 +2598,7 @@ public void Sort_Result_By_Date_Field(bool withFacets) //note: dates internally are stored as Long, see DateTimeType var sc1 = sc.Field("parentID", 1143).OrderBy(new SortableField("updateDate", SortType.Long)); - if (withFacets) + if (HasFacets(withFacets)) { var results1 = sc1 .WithFacets(facets => facets @@ -2077,17 +2637,31 @@ public void Sort_Result_By_Date_Field(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Sort_Result_By_Single_Field(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Sort_Result_By_Single_Field(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullTextSortable)) - : new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FullTextSortable)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullTextSortable)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullTextSortable)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FullTextSortable)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, //Ensure it's set to a fulltextsortable, otherwise it's not sortable fieldDefinitionCollection)) @@ -2113,7 +2687,7 @@ public void Sort_Result_By_Single_Field(bool withFacets) var sc2 = sc.Field("writerName", "administrator") .OrderByDescending(new SortableField("nodeName", SortType.String)); - if (withFacets) + if (HasFacets(withFacets)) { var results1 = sc1.WithFacets(facets => facets.Facet("nodeName")).Execute(); var results2 = sc2.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -2232,21 +2806,35 @@ public void Sort_Result_By_Double_Fields(string fieldType, SortType sortType, bo } } - [TestCase(true)] - [TestCase(false)] - public void Sort_Result_By_Multiple_Fields(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Sort_Result_By_Multiple_Fields(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection( - new FieldDefinition("field1", FieldDefinitionTypes.FacetDouble), - new FieldDefinition("field2", FieldDefinitionTypes.FacetInteger)) - : new FieldDefinitionCollection( + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("field1", FieldDefinitionTypes.FacetTaxonomyDouble), + new FieldDefinition("field2", FieldDefinitionTypes.FacetTaxonomyInteger)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("field1", FieldDefinitionTypes.FacetDouble), + new FieldDefinition("field2", FieldDefinitionTypes.FacetInteger)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection( new FieldDefinition("field1", FieldDefinitionTypes.Double), new FieldDefinition("field2", FieldDefinitionTypes.Integer)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -2267,7 +2855,7 @@ public void Sort_Result_By_Multiple_Fields(bool withFacets) .OrderByDescending(new SortableField("field2", SortType.Int)) .OrderBy(new SortableField("field1", SortType.Double)); - if (withFacets) + if (HasFacets(withFacets)) { var results1 = sc1.WithFacets(facets => facets.Facet("field1").Facet("field2")).Execute(); @@ -2299,16 +2887,31 @@ public void Sort_Result_By_Multiple_Fields(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Standard_Results_Sorted_By_Score(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Standard_Results_Sorted_By_Score(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("bodyText", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText), new FieldDefinition("bodyText", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText), new FieldDefinition("bodyText", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + //Ensure it's set to a fulltextsortable, otherwise it's not sortable + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -2326,7 +2929,7 @@ public void Standard_Results_Sorted_By_Score(bool withFacets) var sc = searcher.CreateQuery("content", BooleanOperation.Or); var sc1 = sc.Field("nodeName", "umbraco").Or().Field("headerText", "umbraco").Or().Field("bodyText", "umbraco"); - if (withFacets) + if (HasFacets(withFacets)) { var results = sc1.WithFacets(facets => facets.Facet("bodyText")).Execute(); @@ -2366,16 +2969,30 @@ public void Standard_Results_Sorted_By_Score(bool withFacets) } - [TestCase(true)] - [TestCase(false)] - public void Skip_Results_Returns_Different_Results(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Skip_Results_Returns_Different_Results(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -2393,7 +3010,7 @@ public void Skip_Results_Returns_Different_Results(bool withFacets) //Arrange var sc = searcher.CreateQuery("content").Field("writerName", "administrator"); - if (withFacets) + if (HasFacets(withFacets)) { //Act var results = sc.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -2415,16 +3032,29 @@ public void Skip_Results_Returns_Different_Results(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Escaping_Includes_All_Words(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Escaping_Includes_All_Words(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -2444,7 +3074,7 @@ public void Escaping_Includes_All_Words(bool withFacets) Console.WriteLine(sc.ToString()); - if (withFacets) + if (HasFacets(withFacets)) { //Act var results = sc.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -2470,16 +3100,29 @@ public void Escaping_Includes_All_Words(bool withFacets) } - [TestCase(true)] - [TestCase(false)] - public void Grouped_And_Examiness(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Grouped_And_Examiness(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -2503,7 +3146,7 @@ public void Grouped_And_Examiness(bool withFacets) new[] { "CWS".MultipleCharacterWildcard(), "A".MultipleCharacterWildcard() }); - if (withFacets) + if (HasFacets(withFacets)) { //Act var results = filter.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -2525,16 +3168,29 @@ public void Grouped_And_Examiness(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Examiness_Proximity(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Examiness_Proximity(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -2555,7 +3211,7 @@ public void Examiness_Proximity(bool withFacets) //get all nodes that contain the words warren and creative within 5 words of each other var filter = criteria.Field("metaKeywords", "Warren creative".Proximity(5)); - if (withFacets) + if (HasFacets(withFacets)) { //Act var results = filter.WithFacets(facets => facets.Facet("nodeName")).Execute(); @@ -2590,18 +3246,31 @@ public void Examiness_Proximity(bool withFacets) /// /// test range query with a Float structure /// - [TestCase(true)] - [TestCase(false)] - public void Float_Range_SimpleIndexSet(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Float_Range_SimpleIndexSet(FacetTestType withFacets) { + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeFloat", FieldDefinitionTypes.FacetTaxonomyFloat)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeFloat", FieldDefinitionTypes.FacetFloat)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeFloat", FieldDefinitionTypes.Float)); + break; + } - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("SomeFloat", FieldDefinitionTypes.FacetFloat)) - : new FieldDefinitionCollection(new FieldDefinition("SomeFloat", FieldDefinitionTypes.Float)); var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, //Ensure it's set to a float fieldDefinitionCollection)) @@ -2628,7 +3297,7 @@ public void Float_Range_SimpleIndexSet(bool withFacets) var criteria2 = searcher.CreateQuery(); var filter2 = criteria2.RangeQuery(new[] { "SomeFloat" }, 101f, 200f, true, true); - if (withFacets) + if (HasFacets(withFacets)) { //Act @@ -2672,19 +3341,32 @@ public void Float_Range_SimpleIndexSet(bool withFacets) /// /// test range query with a Number structure /// - [TestCase(true)] - [TestCase(false)] - public void Number_Range_SimpleIndexSet(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Number_Range_SimpleIndexSet(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("SomeNumber", FieldDefinitionTypes.FacetInteger)) - : new FieldDefinitionCollection(new FieldDefinition("SomeNumber", FieldDefinitionTypes.Integer)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeNumber", FieldDefinitionTypes.FacetTaxonomyInteger)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeNumber", FieldDefinitionTypes.FacetInteger)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeNumber", FieldDefinitionTypes.Integer)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, - //Ensure it's set to a float fieldDefinitionCollection)) { @@ -2709,7 +3391,7 @@ public void Number_Range_SimpleIndexSet(bool withFacets) var criteria2 = searcher.CreateQuery(); var filter2 = criteria2.RangeQuery(new[] { "SomeNumber" }, 101, 200, true, true); - if (withFacets) + if (HasFacets(withFacets)) { //Act var results1 = filter1.WithFacets(facets => facets.Facet("SomeNumber", config => config.MaxCount(1))).Execute(); @@ -2740,17 +3422,31 @@ public void Number_Range_SimpleIndexSet(bool withFacets) /// /// test range query with a Number structure /// - [TestCase(true)] - [TestCase(false)] - public void Double_Range_SimpleIndexSet(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Double_Range_SimpleIndexSet(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("SomeDouble", FieldDefinitionTypes.FacetDouble)) - : new FieldDefinitionCollection(new FieldDefinition("SomeDouble", FieldDefinitionTypes.Double)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeDouble", FieldDefinitionTypes.FacetTaxonomyDouble)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeDouble", FieldDefinitionTypes.FacetDouble)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeDouble", FieldDefinitionTypes.Double)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, //Ensure it's set to a float fieldDefinitionCollection)) @@ -2777,7 +3473,7 @@ public void Double_Range_SimpleIndexSet(bool withFacets) var criteria2 = searcher.CreateQuery("content"); var filter2 = criteria2.RangeQuery(new[] { "SomeDouble" }, 101d, 200d, true, true); - if (withFacets) + if (HasFacets(withFacets)) { //Act var results1 = filter1.WithFacets(facets => facets.Facet("SomeDouble", new DoubleRange[] @@ -2818,19 +3514,32 @@ public void Double_Range_SimpleIndexSet(bool withFacets) /// /// test range query with a Double structure /// - [TestCase(true)] - [TestCase(false)] - public void Long_Range_SimpleIndexSet(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Long_Range_SimpleIndexSet(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("SomeLong", FieldDefinitionTypes.FacetLong)) - : new FieldDefinitionCollection(new FieldDefinition("SomeLong", FieldDefinitionTypes.Long)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeLong", FieldDefinitionTypes.FacetTaxonomyLong)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeLong", FieldDefinitionTypes.FacetLong)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("SomeLong", FieldDefinitionTypes.Long)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, - //Ensure it's set to a float fieldDefinitionCollection)) { indexer.IndexItems(new[] { @@ -2853,7 +3562,7 @@ public void Long_Range_SimpleIndexSet(bool withFacets) var criteria2 = searcher.CreateQuery(); var filter2 = criteria2.RangeQuery(new[] { "SomeLong" }, 101L, 200L, true, true); - if (withFacets) + if (HasFacets(withFacets)) { //Act var results1 = filter1.WithFacets(facets => facets.Facet("SomeLong", new Int64Range[] @@ -2896,19 +3605,33 @@ public void Long_Range_SimpleIndexSet(bool withFacets) /// /// Test range query with a DateTime structure /// - [TestCase(true)] - [TestCase(false)] - public void Date_Range_SimpleIndexSet(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Date_Range_SimpleIndexSet(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("DateCreated", FieldDefinitionTypes.FacetDateTime)) - : new FieldDefinitionCollection(new FieldDefinition("DateCreated", FieldDefinitionTypes.DateTime)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("DateCreated", FieldDefinitionTypes.FacetTaxonomyDateTime)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("DateCreated", FieldDefinitionTypes.FacetDateTime)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("DateCreated", FieldDefinitionTypes.DateTime)); + break; + } + var reIndexDateTime = DateTime.Now; var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -2931,7 +3654,7 @@ public void Date_Range_SimpleIndexSet(bool withFacets) var criteria2 = searcher.CreateQuery(); var filter2 = criteria2.RangeQuery(new[] { "DateCreated" }, reIndexDateTime.AddDays(-1), reIndexDateTime.AddSeconds(-1), true, true); - if (withFacets) + if (HasFacets(withFacets)) { ////Act var results = filter.WithFacets(facets => facets.Facet("DateCreated", new Int64Range[] @@ -2971,16 +3694,30 @@ public void Date_Range_SimpleIndexSet(bool withFacets) } - [TestCase(true)] - [TestCase(false)] - public void Fuzzy_Search(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Fuzzy_Search(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("Content", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("Content", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("Content", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new EnglishAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -3004,7 +3741,7 @@ public void Fuzzy_Search(bool withFacets) Console.WriteLine(filter); Console.WriteLine(filter2); - if (withFacets) + if (HasFacets(withFacets)) { ////Act var results = filter.WithFacets(facets => facets.Facet("Content")).Execute(); @@ -3097,7 +3834,7 @@ public void Execute_With_Take_Max_Results() { for (int i = 0; i < 1000; i++) { - indexer.IndexItems(new[] {ValueSet.FromObject(i.ToString(), "content", new { Content = "hello world" })}); + indexer.IndexItems(new[] { ValueSet.FromObject(i.ToString(), "content", new { Content = "hello world" }) }); } indexer.IndexItems(new[] { ValueSet.FromObject(2000.ToString(), "content", new { Content = "donotfind" }) }); @@ -3117,18 +3854,29 @@ public void Execute_With_Take_Max_Results() } - [TestCase(true)] - [TestCase(false)] - public void Inner_Or_Query(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Inner_Or_Query(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetFullText)); + break; + } var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) - - + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { @@ -3153,7 +3901,7 @@ public void Inner_Or_Query(bool withFacets) var filter = criteria.Field("Type", "type1") .And(query => query.Field("Content", "world").Or().Field("Content", "something"), BooleanOperation.Or); - if (withFacets) + if (HasFacets(withFacets)) { //Act var results = filter.WithFacets(facets => facets.Facet("Type")).Execute(); @@ -3175,18 +3923,30 @@ public void Inner_Or_Query(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Inner_And_Query(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Inner_And_Query(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) - - + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { @@ -3213,7 +3973,7 @@ public void Inner_And_Query(bool withFacets) var filter = criteria.Field("Type", "type1") .And(query => query.Field("Content", "world").And().Field("Content", "hello")); - if (withFacets) + if (HasFacets(withFacets)) { //Act var results = filter.WithFacets(facets => facets.Facet("Type")).Execute(); @@ -3235,18 +3995,30 @@ public void Inner_And_Query(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Inner_Not_Query(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Inner_Not_Query(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) - - + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { @@ -3273,7 +4045,7 @@ public void Inner_Not_Query(bool withFacets) var filter = criteria.Field("Type", "type1") .And(query => query.Field("Content", "world").Not().Field("Content", "something")); - if (withFacets) + if (HasFacets(withFacets)) { //Act var results = filter.WithFacets(facets => facets.Facet("Type")).Execute(); @@ -3295,16 +4067,30 @@ public void Inner_Not_Query(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Complex_Or_Group_Nested_Query(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Complex_Or_Group_Nested_Query(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("Type", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -3339,7 +4125,7 @@ public void Complex_Or_Group_Nested_Query(bool withFacets) Console.WriteLine(filter); - if (withFacets) + if (HasFacets(withFacets)) { //Act @@ -3372,13 +4158,18 @@ public void Complex_Or_Group_Nested_Query(bool withFacets) } - [TestCase(true)] - [TestCase(false)] - public void Custom_Lucene_Query_With_Native(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Custom_Lucene_Query_With_Native(FacetTestType withFacets) { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer)) { var searcher = indexer.Searcher; var criteria = (LuceneSearchQuery)searcher.CreateQuery(); @@ -3386,7 +4177,7 @@ public void Custom_Lucene_Query_With_Native(bool withFacets) //combine a custom lucene query with raw lucene query var op = criteria.NativeQuery("hello:world").And(); - if (withFacets) + if (HasFacets(withFacets)) { criteria.LuceneQuery(NumericRangeQuery.NewInt64Range("numTest", 4, 5, true, true)).WithFacets(facets => facets.Facet("SomeFacet")); } @@ -3497,17 +4288,30 @@ public void Category() //} - [TestCase(true)] - [TestCase(false)] - public void Select_Field(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Select_Field(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) - + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { new ValueSet(1.ToString(), "content", @@ -3532,7 +4336,7 @@ public void Select_Field(bool withFacets) var sc = searcher.CreateQuery("content"); var sc1 = sc.Field("nodeName", "my name 1").SelectField("__Path"); - if (withFacets) + if (HasFacets(withFacets)) { var results = sc1.WithFacets(facets => facets.Facet("nodeName")).Execute(); var facetResults = results.GetFacet("nodeName"); @@ -3555,17 +4359,29 @@ public void Select_Field(bool withFacets) } } - [TestCase(true)] - [TestCase(false)] - public void Select_Fields(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Select_Fields(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) - + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { new ValueSet(1.ToString(), "content", @@ -3590,7 +4406,7 @@ public void Select_Fields(bool withFacets) var sc = searcher.CreateQuery("content"); var sc1 = sc.Field("nodeName", "my name 1").SelectFields(new HashSet(new[] { "nodeName", "bodyText", "id", "__NodeId" })); - if (withFacets) + if (HasFacets(withFacets)) { var results = sc1.WithFacets(facets => facets.Facet("nodeName")).Execute(); var facetResults = results.GetFacet("nodeName"); @@ -3614,17 +4430,29 @@ public void Select_Fields(bool withFacets) } - [TestCase(true)] - [TestCase(false)] - public void Select_Fields_HashSet(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Select_Fields_HashSet(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)); + break; + } var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) - + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { new ValueSet(1.ToString(), "content", @@ -3649,7 +4477,7 @@ public void Select_Fields_HashSet(bool withFacets) var sc = searcher.CreateQuery("content"); var sc1 = sc.Field("nodeName", "my name 1").SelectFields(new HashSet(new string[] { "nodeName", "bodyText" })); - if (withFacets) + if (HasFacets(withFacets)) { var results = sc1.WithFacets(facets => facets.Facet("nodeName")).Execute(); var facetResults = results.GetFacet("nodeName"); @@ -3747,16 +4575,29 @@ public void Can_Skip() } } - [TestCase(true)] - [TestCase(false)] - public void Paging_With_Skip_Take(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Paging_With_Skip_Take(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("writerName", FieldDefinitionTypes.FacetFullText)) - : null; + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("writerName", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("writerName", FieldDefinitionTypes.FacetFullText)); + break; + } var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -3782,7 +4623,7 @@ public void Paging_With_Skip_Take(bool withFacets) int pageSize = 2; //Act - if (withFacets) + if (HasFacets(withFacets)) { var results = sc.WithFacets(facets => facets.Facet("writerName")) @@ -3884,7 +4725,12 @@ public void Given_SkipTake_Returns_ExpectedTotals(int skip, int take, int expect const int indexSize = 5; var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, fieldDefinitionCollection)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + fieldDefinitionCollection)) { var items = Enumerable.Range(0, indexSize).Select(x => ValueSet.FromObject(x.ToString(), "content", new { nodeName = "umbraco", headerText = "world", writerName = "administrator" })); @@ -3920,18 +4766,159 @@ public void Given_SkipTake_Returns_ExpectedTotals(int skip, int take, int expect } } + [Test] + public void SearchAfter_Sorted_Results_Returns_Different_Results() + { + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, analyzer)) + { + indexer.IndexItems(new[] { + ValueSet.FromObject(1.ToString(), "content", + new { nodeName = "umbraco", headerText = "world", writerName = "administrator" }), + ValueSet.FromObject(2.ToString(), "content", + new { nodeName = "umbraco", headerText = "umbraco", writerName = "administrator" }), + ValueSet.FromObject(3.ToString(), "content", + new { nodeName = "umbraco", headerText = "umbraco", writerName = "administrator" }), + ValueSet.FromObject(4.ToString(), "content", + new { nodeName = "umbraco", headerText = "nz", writerName = "administrator" }), + ValueSet.FromObject(5.ToString(), "content", + new { nodeName = "hello", headerText = "world", writerName = "blah" }) + }); + + var searcher = indexer.Searcher; + + //Arrange + var sc = searcher.CreateQuery("content") + .Field("writerName", "administrator") + .OrderByDescending(new SortableField("id", SortType.Int)); + var luceneOptions = new LuceneQueryOptions(0, 2); + //Act + + //There are 4 results + // First query skips 0 and takes 2. + var luceneResults = sc.ExecuteWithLucene(luceneOptions); + Assert.IsNotNull(luceneResults); + Assert.IsNotNull(luceneResults.SearchAfter, "Search After details should be available"); + var luceneResults1List = luceneResults.ToList(); + Assert.IsTrue(luceneResults1List.Any(x => x.Id == "1")); + Assert.IsTrue(luceneResults1List.Any(x => x.Id == "2")); + + // Second query result continues after result 1 (zero indexed), Takes 1, should not include any of the results before or include the SearchAfter docid / scoreid + var searchAfter = new SearchAfterOptions(luceneResults.SearchAfter.DocumentId, + luceneResults.SearchAfter.DocumentScore, + luceneResults.SearchAfter.Fields, + luceneResults.SearchAfter.ShardIndex.Value); + var luceneOptions2 = new LuceneQueryOptions(0, 1, searchAfter); + var luceneResults2 = sc.ExecuteWithLucene(luceneOptions2); + var luceneResults2List = luceneResults2.ToList(); + Assert.IsTrue(luceneResults2List.Any(x => x.Id == "3"), $"Expected to contain next result after docId {luceneResults.SearchAfter.DocumentId}"); + Assert.IsNotNull(luceneResults2); + + Assert.IsFalse(luceneResults2List.Any(x => luceneResults.ToList().Any(y => y.Id == x.Id)), "Results should not overlap"); + + // Third query result continues after result 2 (zero indexed), Takes 1 + var searchAfter2 = new SearchAfterOptions(luceneResults2.SearchAfter.DocumentId, luceneResults2.SearchAfter.DocumentScore, luceneResults2.SearchAfter.Fields, luceneResults2.SearchAfter.ShardIndex.Value); + var luceneOptions3 = new LuceneQueryOptions(0, 1, searchAfter2); + var luceneResults3 = sc.ExecuteWithLucene(luceneOptions3); + Assert.IsNotNull(luceneResults3); + var luceneResults3List = luceneResults3.ToList(); + Assert.IsTrue(luceneResults3List.Any(x => x.Id == "4"), $"Expected to contain next result after docId {luceneResults2.SearchAfter.DocumentId}"); + Assert.IsFalse(luceneResults3.ToList().Any(x => luceneResults2.Any(y => y.Id == x.Id)), "Results should not overlap"); + Assert.IsFalse(luceneResults3.ToList().Any(x => luceneResults.Any(y => y.Id == x.Id)), "Results should not overlap"); + + Assert.AreNotEqual(luceneResults.First().Id, luceneResults2.First().Id, "Results should be different"); + } + } + + [Test] + public void SearchAfter_NonSorted_Results_Returns_Different_Results() + { + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, analyzer)) + { + indexer.IndexItems(new[] { + ValueSet.FromObject(1.ToString(), "content", + new { nodeName = "umbraco", headerText = "world", writerName = "administrator" }), + ValueSet.FromObject(2.ToString(), "content", + new { nodeName = "umbraco", headerText = "umbraco", writerName = "administrator" }), + ValueSet.FromObject(3.ToString(), "content", + new { nodeName = "umbraco", headerText = "umbraco", writerName = "administrator" }), + ValueSet.FromObject(4.ToString(), "content", + new { nodeName = "umbraco", headerText = "nz", writerName = "administrator" }), + ValueSet.FromObject(5.ToString(), "content", + new { nodeName = "hello", headerText = "world", writerName = "blah" }) + }); + + var searcher = indexer.Searcher; + + //Arrange + var sc = searcher.CreateQuery("content") + .Field("writerName", "administrator"); + var luceneOptions = new LuceneQueryOptions(0, 2); + //Act + + //There are 4 results + // First query skips 0 and takes 2. + var luceneResults = sc.ExecuteWithLucene(luceneOptions); + Assert.IsNotNull(luceneResults); + Assert.IsNotNull(luceneResults.SearchAfter, "Search After details should be available"); + var luceneResults1List = luceneResults.ToList(); + Assert.IsTrue(luceneResults1List.Any(x => x.Id == "1")); + Assert.IsTrue(luceneResults1List.Any(x => x.Id == "2")); + + // Second query result continues after result 1 (zero indexed), Takes 1, should not include any of the results before or include the SearchAfter docid / scoreid + var searchAfter = new SearchAfterOptions(luceneResults.SearchAfter.DocumentId, + luceneResults.SearchAfter.DocumentScore, + luceneResults.SearchAfter.Fields, + luceneResults.SearchAfter.ShardIndex.Value); + var luceneOptions2 = new LuceneQueryOptions(0, 1, searchAfter); + var luceneResults2 = sc.ExecuteWithLucene(luceneOptions2); + var luceneResults2List = luceneResults2.ToList(); + Assert.IsTrue(luceneResults2List.Any(x => x.Id == "3"), $"Expected to contain next result after docId {luceneResults.SearchAfter.DocumentId}"); + Assert.IsNotNull(luceneResults2); + + Assert.IsFalse(luceneResults2List.Any(x => luceneResults.ToList().Any(y => y.Id == x.Id)), "Results should not overlap"); + + // Third query result continues after result 2 (zero indexed), Takes 1 + var searchAfter2 = new SearchAfterOptions(luceneResults2.SearchAfter.DocumentId, luceneResults2.SearchAfter.DocumentScore, luceneResults2.SearchAfter.Fields, luceneResults2.SearchAfter.ShardIndex.Value); + var luceneOptions3 = new LuceneQueryOptions(0, 1, searchAfter2); + var luceneResults3 = sc.ExecuteWithLucene(luceneOptions3); + Assert.IsNotNull(luceneResults3); + var luceneResults3List = luceneResults3.ToList(); + Assert.IsTrue(luceneResults3List.Any(x => x.Id == "4"), $"Expected to contain next result after docId {luceneResults2.SearchAfter.DocumentId}"); + Assert.IsFalse(luceneResults3.ToList().Any(x => luceneResults2.Any(y => y.Id == x.Id)), "Results should not overlap"); + Assert.IsFalse(luceneResults3.ToList().Any(x => luceneResults.Any(y => y.Id == x.Id)), "Results should not overlap"); + + Assert.AreNotEqual(luceneResults.First().Id, luceneResults2.First().Id, "Results should be different"); + } + } + #if NET6_0_OR_GREATER - [TestCase(true)] - [TestCase(false)] - public void Range_DateOnly(bool withFacets) + [TestCase(FacetTestType.TaxonomyFacets)] [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.NoFacets)] + public void Range_DateOnly(FacetTestType withFacets) { - var fieldDefinitionCollection = withFacets ? - new FieldDefinitionCollection(new FieldDefinition("created", FieldDefinitionTypes.FacetDateTime)) - : new FieldDefinitionCollection(new FieldDefinition("created", FieldDefinitionTypes.DateTime)); + FieldDefinitionCollection fieldDefinitionCollection = null; + switch (withFacets) + { + case FacetTestType.TaxonomyFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("created", FieldDefinitionTypes.FacetTaxonomyDateTime)); + break; + case FacetTestType.SortedSetFacets: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("created", FieldDefinitionTypes.FacetDateTime)); + break; + default: + fieldDefinitionCollection = new FieldDefinitionCollection(new FieldDefinition("created", FieldDefinitionTypes.DateTime)); + break; + } var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex( + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -3968,7 +4955,7 @@ public void Range_DateOnly(bool withFacets) var numberSortedCriteria = searcher.CreateQuery() .RangeQuery(new[] { "created" }, new DateOnly(2000, 01, 02), new DateOnly(2000, 01, 05), maxInclusive: false); - if (withFacets) + if (HasFacets(withFacets)) { var numberSortedResult = numberSortedCriteria.WithFacets(facets => facets.Facet("created")).Execute(); var facetResult = numberSortedResult.GetFacet("created"); @@ -4083,5 +5070,121 @@ public void Range_DateOnly_No_Inclusive() } } #endif + + [TestCase(1, 2, 1, 2)] + [TestCase(2, 2, 2, 2)] + public void GivenSearchAfterTake_Returns_ExpectedTotals_Facet(int firstTake, int secondTake, int expectedFirstResultCount, int expectedSecondResultCount) + { + const int indexSize = 5; + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, analyzer, new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)))) + { + var items = Enumerable.Range(0, indexSize).Select(x => ValueSet.FromObject(x.ToString(), "content", + new { nodeName = "umbraco", headerText = "world", writerName = "administrator" })); + + indexer.IndexItems(items); + + var searcher = indexer.Searcher; + + //Arrange + + var sc = searcher.CreateQuery("content") + .Field("writerName", "administrator") + .WithFacets(facets => facets.Facet("nodeName")); + + //Act + + var results1 = sc.ExecuteWithLucene(new LuceneQueryOptions(0, firstTake)); + + var facetResults1 = results1.GetFacet("nodeName"); + + Assert.AreEqual(indexSize, results1.TotalItemCount); + Assert.AreEqual(expectedFirstResultCount, results1.Count()); + Assert.AreEqual(1, facetResults1.Count()); + Assert.AreEqual(5, facetResults1.Facet("umbraco").Value); + + Assert.IsNotNull(results1); + + var results2 = sc.Execute(new LuceneQueryOptions(0, secondTake, results1.SearchAfter)); + + var facetResults2 = results2.GetFacet("nodeName"); + + Assert.AreEqual(indexSize, results2.TotalItemCount); + Assert.AreEqual(expectedSecondResultCount, results2.Count()); + Assert.AreEqual(1, facetResults2.Count()); + Assert.AreEqual(5, facetResults2.Facet("umbraco").Value); + var firstResults = results1.ToArray(); + var secondResults = results2.ToArray(); + Assert.IsFalse(firstResults.Any(x => secondResults.Any(y => y.Id == x.Id)), "The second set of results should not contain the first set of results"); + + } + } + [TestCase(1, 2, 1, 2)] + [TestCase(2, 2, 2, 2)] + public void GivenTaxonomyIndexSearchAfterTake_Returns_ExpectedTotals_Facet(int firstTake, int secondTake, int expectedFirstResultCount, int expectedSecondResultCount) + { + const int indexSize = 5; + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + var facetConfigs = new FacetsConfig(); + facetConfigs.SetIndexFieldName("taxonomynodeName", "taxonomy_nodeName"); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTaxonomyTestIndex(luceneDir, luceneTaxonomyDir, analyzer, new FieldDefinitionCollection( + new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText), + new FieldDefinition("taxonomynodeName", FieldDefinitionTypes.FacetTaxonomyFullText) + + ), facetsConfig: facetConfigs)) + { + var items = Enumerable.Range(0, indexSize).Select(x => ValueSet.FromObject(x.ToString(), "content", + new { nodeName = "umbraco", headerText = "world", writerName = "administrator", taxonomynodeName = "umbraco" })); + + indexer.IndexItems(items); + + var taxonomySearcher = indexer.TaxonomySearcher; + var taxonomyCategoryCount = taxonomySearcher.CategoryCount; + + //Arrange + + var sc = taxonomySearcher.CreateQuery("content") + .Field("writerName", "administrator") + .WithFacets(facets => + { + facets.Facet("nodeName"); + facets.Facet("taxonomynodeName"); + }); + + //Act + + var results1 = sc.ExecuteWithLucene(new LuceneQueryOptions(0, firstTake)); + + var facetResults1 = results1.GetFacet("nodeName"); + + Assert.AreEqual(indexSize, results1.TotalItemCount); + Assert.AreEqual(expectedFirstResultCount, results1.Count()); + Assert.AreEqual(1, facetResults1.Count()); + Assert.AreEqual(5, facetResults1.Facet("umbraco").Value); + + Assert.IsNotNull(results1); + + var facetTaxonomyResults1 = results1.GetFacet("taxonomynodeName"); + Assert.AreEqual(1, facetTaxonomyResults1.Count()); + Assert.AreEqual(5, facetTaxonomyResults1.Facet("umbraco").Value); + + var results2 = sc.Execute(new LuceneQueryOptions(0, secondTake, results1.SearchAfter)); + + var facetResults2 = results2.GetFacet("nodeName"); + var facetTaxonomyResults2 = results2.GetFacet("taxonomynodeName"); + + Assert.AreEqual(indexSize, results2.TotalItemCount); + Assert.AreEqual(expectedSecondResultCount, results2.Count()); + Assert.AreEqual(1, facetResults2.Count()); + Assert.AreEqual(5, facetResults2.Facet("umbraco").Value); + var firstResults = results1.ToArray(); + var secondResults = results2.ToArray(); + Assert.IsFalse(firstResults.Any(x => secondResults.Any(y => y.Id == x.Id)), "The second set of results should not contain the first set of results"); + + } + } } } diff --git a/src/Examine.Test/ExamineBaseTest.cs b/src/Examine.Test/ExamineBaseTest.cs index 9a2e1bb25..d15fb2590 100644 --- a/src/Examine.Test/ExamineBaseTest.cs +++ b/src/Examine.Test/ExamineBaseTest.cs @@ -29,7 +29,7 @@ public TestIndex GetTestIndex(Directory d, Analyzer analyzer, FieldDefinitionCol Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions { FieldDefinitions = fieldDefinitions, - DirectoryFactory = new GenericDirectoryFactory(_ => d), + DirectoryFactory = new GenericDirectoryFactory(_ => d, null), Analyzer = analyzer, IndexDeletionPolicy = indexDeletionPolicy, IndexValueTypesFactory = indexValueTypesFactory, @@ -45,5 +45,23 @@ public TestIndex GetTestIndex(IndexWriter writer) Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneIndexOptions()), writer); } + + public TestIndex GetTaxonomyTestIndex(Directory d, Directory taxonomyDirectory, Analyzer analyzer, FieldDefinitionCollection fieldDefinitions = null, IndexDeletionPolicy indexDeletionPolicy = null, IReadOnlyDictionary indexValueTypesFactory = null, FacetsConfig? facetsConfig = null) + { + var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug)); + return new TestIndex( + loggerFactory, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions + { + FieldDefinitions = fieldDefinitions, + DirectoryFactory = new GenericDirectoryFactory(_ => d, _ => taxonomyDirectory), + Analyzer = analyzer, + IndexDeletionPolicy = indexDeletionPolicy, + IndexValueTypesFactory = indexValueTypesFactory, + FacetsConfig = facetsConfig ?? new FacetsConfig(), + UseTaxonomyIndex = true + })); + } + } } diff --git a/src/Examine.Web.Demo/ConfigureIndexOptions.cs b/src/Examine.Web.Demo/ConfigureIndexOptions.cs index c36999822..a35bafe5c 100644 --- a/src/Examine.Web.Demo/ConfigureIndexOptions.cs +++ b/src/Examine.Web.Demo/ConfigureIndexOptions.cs @@ -1,12 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Text.RegularExpressions; using Examine.Lucene; using Examine.Lucene.Analyzers; -using Examine.Lucene.Directories; using Examine.Lucene.Indexing; -using Lucene.Net.Index; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Examine.Web.Demo @@ -44,6 +38,22 @@ public void Configure(string name, LuceneDirectoryIndexOptions options) // to a Value Type called "phone" defined above. options.FieldDefinitions.AddOrUpdate(new FieldDefinition("phone", "phone")); break; + case "TaxonomyFacetIndex": + options.UseTaxonomyIndex = true; + options.FacetsConfig.SetMultiValued("Tags", true); + options.FieldDefinitions.AddOrUpdate(new FieldDefinition("AddressState", FieldDefinitionTypes.FacetTaxonomyFullText)); + options.FieldDefinitions.AddOrUpdate(new FieldDefinition("AddressStateCity", FieldDefinitionTypes.FacetTaxonomyFullText)); + options.FieldDefinitions.AddOrUpdate(new FieldDefinition("Tags", FieldDefinitionTypes.FacetTaxonomyFullText)); + break; + + case "FacetIndex": + options.UseTaxonomyIndex = false; + options.FacetsConfig.SetMultiValued("Tags", true); + options.FieldDefinitions.AddOrUpdate(new FieldDefinition("AddressState", FieldDefinitionTypes.FacetFullText)); + options.FieldDefinitions.AddOrUpdate(new FieldDefinition("AddressStateCity", FieldDefinitionTypes.FacetFullText)); + options.FieldDefinitions.AddOrUpdate(new FieldDefinition("Tags", FieldDefinitionTypes.FacetFullText)); + break; + } } diff --git a/src/Examine.Web.Demo/Data/BogusDataService.cs b/src/Examine.Web.Demo/Data/BogusDataService.cs index 96d1daa81..84d8ed706 100644 --- a/src/Examine.Web.Demo/Data/BogusDataService.cs +++ b/src/Examine.Web.Demo/Data/BogusDataService.cs @@ -1,8 +1,5 @@ -using System; -using System.Collections.Generic; -using System.Linq; using Bogus; -using Lucene.Net.Facet.Range; +using Bogus.DataSets; namespace Examine.Web.Demo.Controllers { @@ -15,15 +12,49 @@ public class BogusDataService public IEnumerable GenerateData(int count) { return Enumerable.Range(1, count) - .Select(x => new Person()) + .Select(x => new Person()) + .Select((person, index) => new ValueSet( + index.ToString(), + "person", + PersonValues(person))); + } + + private IDictionary> PersonValues(Person person) + { + var values = new Dictionary> + { + [nameof(person.FullName)] = new List(1) { person.FullName }, + [nameof(person.Email)] = new List(1) { person.Email }, + [nameof(person.Phone)] = new List(1) { person.Phone }, + [nameof(person.Website)] = new List(1) { person.Website }, + [$"{nameof(person.Company)}{nameof(person.Company.Name)}"] = new List(1) { person.Company.Name }, + [$"{nameof(person.Company)}{nameof(person.Company.CatchPhrase)}"] = new List(1) { person.Company.CatchPhrase }, + [$"{nameof(person.Address)}{nameof(person.Address.City)}"] = new List(1) { person.Address.City }, + [$"{nameof(person.Address)}{nameof(person.Address.State)}"] = new List(1) { person.Address.State }, + [$"{nameof(person.Address)}{nameof(person.Address.Street)}"] = new List(1) { person.Address.Street } + }; + return values; + } + + + /// + /// Return a ton of people + /// + /// + public IEnumerable GenerateFacetData(int count) + { + return Enumerable.Range(1, count) + .Select(x => (new Person(), new Commerce())) .Select((person, index) => new ValueSet( index.ToString(), "person", - PersonValues(person))); + FacetPersonValues(person))); } - private IDictionary> PersonValues(Person person) + private IDictionary> FacetPersonValues((Person person, Commerce commerce) personCommerce) { + var person = personCommerce.person; + var commerce = personCommerce.commerce; var values = new Dictionary> { [nameof(person.FullName)] = new List(1) { person.FullName }, @@ -34,7 +65,11 @@ private IDictionary> PersonValues(Person person) [$"{nameof(person.Company)}{nameof(person.Company.CatchPhrase)}"] = new List(1) { person.Company.CatchPhrase }, [$"{nameof(person.Address)}{nameof(person.Address.City)}"] = new List(1) { person.Address.City }, [$"{nameof(person.Address)}{nameof(person.Address.State)}"] = new List(1) { person.Address.State }, - [$"{nameof(person.Address)}{nameof(person.Address.Street)}"] = new List(1) { person.Address.Street } + [$"{nameof(person.Address)}{nameof(person.Address.Street)}"] = new List(1) { person.Address.Street }, + [$"{nameof(person.Address)}{nameof(person.Address.State)}{nameof(person.Address.City)}"] = new List(2) { + new object[] { person.Address.City, new string[] { person.Address.State, person.Address.City }} + }, + [$"Tags"] = commerce.Categories(3), }; return values; } diff --git a/src/Examine.Web.Demo/Data/IndexService.Facets.cs b/src/Examine.Web.Demo/Data/IndexService.Facets.cs new file mode 100644 index 000000000..a5472f6ee --- /dev/null +++ b/src/Examine.Web.Demo/Data/IndexService.Facets.cs @@ -0,0 +1,7 @@ +namespace Examine.Web.Demo.Data +{ + public partial class IndexService + { + + } +} diff --git a/src/Examine.Web.Demo/Data/IndexService.cs b/src/Examine.Web.Demo/Data/IndexService.cs index ede13380f..8dd6eaa67 100644 --- a/src/Examine.Web.Demo/Data/IndexService.cs +++ b/src/Examine.Web.Demo/Data/IndexService.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using Examine.Lucene.Providers; +using Examine.Lucene.Search; using Examine.Search; using Examine.Web.Demo.Controllers; using Examine.Web.Demo.Data.Models; @@ -7,12 +8,13 @@ namespace Examine.Web.Demo.Data { - public class IndexService + public partial class IndexService { private readonly IExamineManager _examineManager; private readonly BogusDataService _bogusDataService; - public IndexService(IExamineManager examineManager, BogusDataService bogusDataService) { + public IndexService(IExamineManager examineManager, BogusDataService bogusDataService) + { _examineManager = examineManager; _bogusDataService = bogusDataService; } @@ -23,7 +25,15 @@ public void RebuildIndex(string indexName, int dataSize) index.CreateIndex(); - var data = _bogusDataService.GenerateData(dataSize); + IEnumerable data; + if (index.Name.Contains("Facet")) + { + data = _bogusDataService.GenerateFacetData(dataSize); + } + else + { + data = _bogusDataService.GenerateData(dataSize); + } index.IndexItems(data); } @@ -98,6 +108,17 @@ private IIndex GetIndex(string indexName) return index; } + + public ILuceneSearchResults SearchLucene(string indexName, Func queryBuilder, QueryOptions queryOptions) + { + var index = GetIndex(indexName); + + var searcher = index.Searcher; + var criteria = searcher.CreateQuery(); + var finalCriteria = queryBuilder(criteria); + return finalCriteria.ExecuteWithLucene(queryOptions); + } + } } diff --git a/src/Examine.Web.Demo/Examine.Web.Demo.csproj b/src/Examine.Web.Demo/Examine.Web.Demo.csproj index 2a31a31ae..1b9b52298 100644 --- a/src/Examine.Web.Demo/Examine.Web.Demo.csproj +++ b/src/Examine.Web.Demo/Examine.Web.Demo.csproj @@ -6,6 +6,14 @@ enable + + + + + + + + @@ -14,4 +22,8 @@ + + + + diff --git a/src/Examine.Web.Demo/IndexFactoryExtensions.cs b/src/Examine.Web.Demo/IndexFactoryExtensions.cs index 39c1d6e98..36688fc33 100644 --- a/src/Examine.Web.Demo/IndexFactoryExtensions.cs +++ b/src/Examine.Web.Demo/IndexFactoryExtensions.cs @@ -1,3 +1,4 @@ +using Lucene.Net.Facet; using Microsoft.Extensions.DependencyInjection; namespace Examine.Web.Demo @@ -13,9 +14,31 @@ public static IServiceCollection CreateIndexes(this IServiceCollection services) services.AddExamineLuceneIndex("SyncedIndex"); + var taxonomyFacetIndexFacetsConfig = new FacetsConfig(); + taxonomyFacetIndexFacetsConfig.SetIndexFieldName("AddressState", "AddressState"); + + taxonomyFacetIndexFacetsConfig.SetIndexFieldName("AddressStateCity", "AddressStateCity"); + taxonomyFacetIndexFacetsConfig.SetHierarchical("AddressStateCity", true); + taxonomyFacetIndexFacetsConfig.SetMultiValued("AddressStateCity", false); + + taxonomyFacetIndexFacetsConfig.SetIndexFieldName("Tags", "Tags"); + taxonomyFacetIndexFacetsConfig.SetMultiValued("Tags", true); + + services.AddExamineLuceneIndex( + "TaxonomyFacetIndex", + facetsConfig: taxonomyFacetIndexFacetsConfig); + + var facetIndexFacetsConfig = new FacetsConfig(); + + services.AddExamineLuceneIndex( + "FacetIndex", + facetsConfig: facetIndexFacetsConfig); + + services.AddExamineLuceneMultiSearcher( "MultiIndexSearcher", - new[] { "MyIndex", "SyncedIndex" }); + new[] { "MyIndex", "SyncedIndex", "FacetIndex" }, + facetsConfig: new FacetsConfig()); services.ConfigureOptions(); diff --git a/src/Examine.Web.Demo/Pages/FacetSearch.razor b/src/Examine.Web.Demo/Pages/FacetSearch.razor new file mode 100644 index 000000000..bba0eb171 --- /dev/null +++ b/src/Examine.Web.Demo/Pages/FacetSearch.razor @@ -0,0 +1,116 @@ +@page "/facetsearch" +@using Examine.Lucene.Search; +@using Examine.Search; +@using Examine.Web.Demo.Data +@using System.Diagnostics +@inject IndexService IndexService +@inject IExamineManager ExamineMgr + +

Faceted Search

+ +

Here you can try searching in the Faceted index created in the demo application.

+ +
Quick searches
+ + + +

Index to search

+ + +
Lucene query
+
+ + +
+ +
+ @if (_searchResults != null && _searchResults is IFacetResults facetSearchResults) + { +

Facets found (Showing @facetSearchResults.Facets.Count())

+ + @foreach (var facet in facetSearchResults.Facets) + { +
@facet.Key
+
    + @foreach (var item in facet.Value) + { +
  • @item.Label (@item.Value)
    • + } +
+ } + } + else + { +

No facet results

+ } +
+ +
+ @if (_searchResults != null && _searchResults is ILuceneSearchResults luceneSearchResults) + { +

@luceneSearchResults.TotalItemCount Results found (Showing @luceneSearchResults.Count()) - Found in: @_searchTime

+ @foreach (var searchResult in luceneSearchResults) + { +

Id: @searchResult.Id, Score: @searchResult.Score, Values: @(string.Join(", ", searchResult.Values.Select(x => $"[{x.Key}: {x.Value}]")))

+ } + } + else if (_searchResults != null) + { +

@_searchResults.TotalItemCount Results found (Showing @_searchResults.Count()) - Found in: @_searchTime

+ @foreach (var searchResult in _searchResults) + { +

Id: @searchResult.Id, Score: @searchResult.Score, Values: @(string.Join(", ", searchResult.Values.Select(x => $"[{x.Key}: {x.Value}]")))

+ } + } + else + { +

No results

+ } +
+ +@code { + private List _indexes = new(); + private string _selectedIndex = string.Empty; + private string _query = string.Empty; + private ISearchResults? _searchResults; + private string _searchTime = string.Empty; + + protected override void OnInitialized() + { + _indexes = IndexService.GetAllIndexes().Where(x => x.Name.Contains("Facet")).ToList(); + _selectedIndex = _indexes.FirstOrDefault(x => x.Name.Contains("TaxonomyFacet"))?.Name ?? _indexes.First().Name; + } + + private void SearchIndex() + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + var luceneSearchResults = IndexService.SearchLucene( + _selectedIndex, + (query) => query.NativeQuery(_query.Trim()).WithFacets(facets => { facets.Facet("AddressState"); facets.Facet("AddressStateCity"); facets.Facet("Tags"); }), + new LuceneQueryOptions(0, 100) + ); + _searchResults = luceneSearchResults; + stopwatch.Stop(); + _searchTime = $"{stopwatch.ElapsedMilliseconds} Milliseconds ({stopwatch.Elapsed:g})"; + } + + private void GetFirst100Items() + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + var luceneSearchResults = IndexService.SearchLucene( + _selectedIndex, + (query) => query.All().WithFacets(facets => { facets.Facet("AddressState"); facets.Facet("AddressStateCity"); facets.Facet("Tags"); }), + new LuceneQueryOptions(0, 100) + ); + _searchResults = luceneSearchResults; + stopwatch.Stop(); + _searchTime = $"{stopwatch.ElapsedMilliseconds} Milliseconds ({stopwatch.Elapsed:g})"; + } +} diff --git a/src/Examine.Web.Demo/Shared/NavMenu.razor b/src/Examine.Web.Demo/Shared/NavMenu.razor index a30776e8b..6997e92a5 100644 --- a/src/Examine.Web.Demo/Shared/NavMenu.razor +++ b/src/Examine.Web.Demo/Shared/NavMenu.razor @@ -23,6 +23,11 @@ Search data + + + Faceted Search +