From 756dc61b5a8c1a18f4c76ce610da754266e39356 Mon Sep 17 00:00:00 2001 From: August Johnson Date: Tue, 2 Apr 2024 19:33:07 -0500 Subject: [PATCH 01/14] Adjusted quite a bit. --- api_v2/management/commands/index_v1.py | 7 +++--- api_v2/migrations/0073_auto_20240403_0010.py | 22 ++++++++++++++++++ api_v2/models/search.py | 23 +------------------ api_v2/serializers/search.py | 24 +++++++++++++------- api_v2/views/search.py | 17 +++++++------- 5 files changed, 51 insertions(+), 42 deletions(-) create mode 100644 api_v2/migrations/0073_auto_20240403_0010.py diff --git a/api_v2/management/commands/index_v1.py b/api_v2/management/commands/index_v1.py index 9054430c..9d1ca3a5 100644 --- a/api_v2/management/commands/index_v1.py +++ b/api_v2/management/commands/index_v1.py @@ -35,10 +35,9 @@ def load_content(self,model,schema): for o in model.objects.all(): search_results.append(v2.SearchResult( document_pk=o.document.slug, - document_name=o.document.title, object_pk=o.slug, object_name=o.name, - object_route=o.route, + object_model=o.__class__.__name__, schema_version="v1", text=o.name+"\n"+o.desc @@ -52,9 +51,9 @@ def load_index(self): cursor.execute("DROP TABLE IF EXISTS search_index;") - cursor.execute("CREATE VIRTUAL TABLE search_index USING FTS5(document_pk,document_name,object_pk,object_name,object_route,text,schema_version);") + cursor.execute("CREATE VIRTUAL TABLE search_index USING FTS5(document_pk,object_pk,object_name,object_model,text,schema_version);") - cursor.execute("INSERT INTO search_index (document_pk,document_name,object_pk,object_name,object_route,text,schema_version) SELECT document_pk,document_name,object_pk,object_name,object_route,text,schema_version FROM api_v2_searchresult") + cursor.execute("INSERT INTO search_index (document_pk,object_pk,object_name,object_model,text,schema_version) SELECT document_pk,object_pk,object_name,object_model,text,schema_version FROM api_v2_searchresult") def check_fts_enabled(self): diff --git a/api_v2/migrations/0073_auto_20240403_0010.py b/api_v2/migrations/0073_auto_20240403_0010.py new file mode 100644 index 00000000..a7298aa1 --- /dev/null +++ b/api_v2/migrations/0073_auto_20240403_0010.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.20 on 2024-04-03 00:10 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api_v2', '0072_merge_20240329_1829'), + ] + + operations = [ + migrations.RenameField( + model_name='searchresult', + old_name='object_route', + new_name='object_model', + ), + migrations.RemoveField( + model_name='searchresult', + name='document_name', + ), + ] diff --git a/api_v2/models/search.py b/api_v2/models/search.py index e2b7289b..7288a302 100644 --- a/api_v2/models/search.py +++ b/api_v2/models/search.py @@ -5,32 +5,11 @@ class SearchResult(models.Model): """ The Search Result object model""" document_pk = models.CharField(max_length=255) - document_name = models.CharField(max_length=100) object_pk = models.CharField(max_length=255) object_name = models.CharField(max_length=100) - object_route = models.CharField(max_length=255) + object_model = models.CharField(max_length=255) schema_version = models.CharField(max_length=100) rank = models.DecimalField(max_digits=10, decimal_places=4, null=True, default=None) text = models.TextField(null=True, default=None) highlighted = models.TextField(null=True, default=None) - - @property - def document_slug(self): - return self.document_pk - - @property - def document_title(self): - return self.document_name - - @property - def route(self): - return self.object_route - - @property - def slug(self): - return self.object_pk - - @property - def name(self): - return self.object_name diff --git a/api_v2/serializers/search.py b/api_v2/serializers/search.py index 16ba1bd2..8fa8f7e2 100644 --- a/api_v2/serializers/search.py +++ b/api_v2/serializers/search.py @@ -10,17 +10,17 @@ class SearchResultSerializer(serializers.ModelSerializer): text = serializers.ReadOnlyField() highlighted = serializers.ReadOnlyField() object = serializers.SerializerMethodField(method_name='get_object') + document = serializers.SerializerMethodField(method_name='get_document') class Meta: model = models.SearchResult fields = [ - 'document_pk', - 'document_name', + 'document', 'object_pk', 'object_name', 'object', - 'object_route', + 'object_model', 'schema_version', 'rank', 'text', @@ -28,18 +28,26 @@ class Meta: def get_object(self, obj): if obj.schema_version == 'v1': - if obj.object_route == 'magicitems/': + if obj.object_model == 'MagicItem': result_detail = v1.MagicItem.objects.get(slug=obj.object_pk) return result_detail.search_result_extra_fields() - if obj.object_route == 'monsters/': + if obj.object_model == 'Monster': result_detail = v1.Monster.objects.get(slug=obj.object_pk) return result_detail.search_result_extra_fields() - if obj.object_route == 'spells/': + if obj.object_model == 'Spell': result_detail = v1.Spell.objects.get(slug=obj.object_pk) return result_detail.search_result_extra_fields() - if obj.object_route == 'sections/': + if obj.object_model == 'Section': result_detail = v1.Section.objects.get(slug=obj.object_pk) - return result_detail.search_result_extra_fields() \ No newline at end of file + return result_detail.search_result_extra_fields() + + def get_document(self, obj): + if obj.schema_version == 'v1': + doc = v1.Document.objects.get(slug=obj.document_pk) + return { + 'document_pk': doc.slug, + 'document_name': doc.title + } diff --git a/api_v2/views/search.py b/api_v2/views/search.py index f7510d84..b845871b 100644 --- a/api_v2/views/search.py +++ b/api_v2/views/search.py @@ -33,20 +33,21 @@ def get_queryset(self): else: document_pk = self.request.query_params.get("document_pk") - if self.request.query_params.get("object_route") is None: - object_route = '%' + if self.request.query_params.get("object_model") is None: + object_model = '%' else: - object_route = self.request.query_params.get("object_route") + object_model = self.request.query_params.get("object_model") - queryset = models.SearchResult.objects.raw( + weighted_queryset = models.SearchResult.objects.raw( "SELECT 1 as id,rank, " + "snippet(search_index,5,'','','...',20) as highlighted, " + "* FROM search_index " + "WHERE " + "schema_version LIKE %s " + "AND document_pk LIKE %s " + - "AND object_route LIKE %s " + - "AND text MATCH %s " + - "ORDER BY rank",[schema_version, document_pk, object_route, query]) + "AND object_model LIKE %s " + + "AND search_index MATCH %s" + + "AND rank MATCH 'bm25(1.0, 1.0, 1.0, 10.0)'"+ # This line results in a 10x weight to Name + "ORDER BY rank",[schema_version, document_pk, object_model, query]) - return queryset + return weighted_queryset From 5d2c38d08f35cf0e229b68e0ac1b605b76ef2330 Mon Sep 17 00:00:00 2001 From: August Johnson Date: Tue, 2 Apr 2024 19:51:45 -0500 Subject: [PATCH 02/14] First v2 model being indexed. --- api_v2/management/commands/index_v1.py | 69 +++++++++++++++++++------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/api_v2/management/commands/index_v1.py b/api_v2/management/commands/index_v1.py index 9d1ca3a5..ba3e8381 100644 --- a/api_v2/management/commands/index_v1.py +++ b/api_v2/management/commands/index_v1.py @@ -18,22 +18,14 @@ def unload_all_content(self): v2.SearchResult.objects.all().delete() print("UNLOADED_OBJECT_COUNT:{}".format(object_count)) - - def load_content(self,model,schema): - print("SCHEMA:{} OBJECT_COUNT:{} MODEL:{} TABLE_NAME:{}".format( - schema, - model.objects.all().count(), - model.__name__, - model._meta.db_table)) - + def load_v1_content(self, model): + results = [] standard_v1_models = ['MagicItem','Spell','Monster','CharClass','Archetype', - 'Race','Subrace','Plane','Section','Feat','Condition','Background','Weapon','Armor'] - - search_results = [] + 'Race','Subrace','Plane','Section','Feat','Condition','Background','Weapon','Armor'] - if model.__name__ in standard_v1_models and schema=='v1': + if model.__name__ in standard_v1_models: for o in model.objects.all(): - search_results.append(v2.SearchResult( + results.append(v2.SearchResult( document_pk=o.document.slug, object_pk=o.slug, object_name=o.name, @@ -42,19 +34,58 @@ def load_content(self,model,schema): text=o.name+"\n"+o.desc )) + return results + + + def load_v2_content(self, model): + results = [] + standard_v2_models = ['Item'] + + if model.__name__ in standard_v2_models: + for o in model.objects.all(): + results.append(v2.SearchResult( + document_pk=o.document.pk, + object_pk=o.pk, + object_name=o.name, + object_model=o.__class__.__name__, + schema_version='v2', + text=o.desc + )) + return results + + + def load_content(self,model,schema): + print("SCHEMA:{} OBJECT_COUNT:{} MODEL:{} TABLE_NAME:{}".format( + schema, + model.objects.all().count(), + model.__name__, + model._meta.db_table)) - v2.SearchResult.objects.bulk_create(search_results) + if schema == 'v1': + v2.SearchResult.objects.bulk_create( + self.load_v1_content(model) + ) + if schema == 'v2': + v2.SearchResult.objects.bulk_create( + self.load_v2_content(model) + ) def load_index(self): with connection.cursor() as cursor: cursor.execute("DROP TABLE IF EXISTS search_index;") - - cursor.execute("CREATE VIRTUAL TABLE search_index USING FTS5(document_pk,object_pk,object_name,object_model,text,schema_version);") - cursor.execute("INSERT INTO search_index (document_pk,object_pk,object_name,object_model,text,schema_version) SELECT document_pk,object_pk,object_name,object_model,text,schema_version FROM api_v2_searchresult") - + cursor.execute( + "CREATE VIRTUAL TABLE search_index " + + "USING FTS5(document_pk,object_pk,object_name,object_model,text,schema_version);") + + cursor.execute( + "INSERT INTO search_index " + + "(document_pk,object_pk,object_name,object_model,text,schema_version) " + + "SELECT document_pk,object_pk,object_name,object_model,text,schema_version " + + "FROM api_v2_searchresult") + def check_fts_enabled(self): #import sqlite3 @@ -90,6 +121,8 @@ def handle(self, *args, **options) -> None: self.load_content(v1.Weapon,"v1") self.load_content(v1.Armor,"v1") + self.load_content(v2.Item,"v2") + # Take the content table's current data and load it into the index. self.load_index() From b171832bcbb991540d9adce6293c2d675a6381e5 Mon Sep 17 00:00:00 2001 From: August Johnson Date: Wed, 3 Apr 2024 18:15:22 -0500 Subject: [PATCH 03/14] some basic refactors. --- api_v2/management/commands/index_v1.py | 4 +--- api_v2/serializers/search.py | 14 +++++++++----- api_v2/views/search.py | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/api_v2/management/commands/index_v1.py b/api_v2/management/commands/index_v1.py index ba3e8381..b2969683 100644 --- a/api_v2/management/commands/index_v1.py +++ b/api_v2/management/commands/index_v1.py @@ -36,7 +36,6 @@ def load_v1_content(self, model): )) return results - def load_v2_content(self, model): results = [] standard_v2_models = ['Item'] @@ -44,7 +43,7 @@ def load_v2_content(self, model): if model.__name__ in standard_v2_models: for o in model.objects.all(): results.append(v2.SearchResult( - document_pk=o.document.pk, + document_pk=o.document.key, object_pk=o.pk, object_name=o.name, object_model=o.__class__.__name__, @@ -53,7 +52,6 @@ def load_v2_content(self, model): )) return results - def load_content(self,model,schema): print("SCHEMA:{} OBJECT_COUNT:{} MODEL:{} TABLE_NAME:{}".format( schema, diff --git a/api_v2/serializers/search.py b/api_v2/serializers/search.py index 8fa8f7e2..c189cd11 100644 --- a/api_v2/serializers/search.py +++ b/api_v2/serializers/search.py @@ -6,9 +6,6 @@ from api import models as v1 class SearchResultSerializer(serializers.ModelSerializer): - rank = serializers.ReadOnlyField() - text = serializers.ReadOnlyField() - highlighted = serializers.ReadOnlyField() object = serializers.SerializerMethodField(method_name='get_object') document = serializers.SerializerMethodField(method_name='get_document') @@ -48,6 +45,13 @@ def get_document(self, obj): if obj.schema_version == 'v1': doc = v1.Document.objects.get(slug=obj.document_pk) return { - 'document_pk': doc.slug, - 'document_name': doc.title + 'key': doc.slug, + 'name': doc.title } + + if obj.schema_version == 'v2': + doc = models.Document.objects.get(key=obj.document_pk) + return { + 'key': doc.key, + 'name': doc.name + } \ No newline at end of file diff --git a/api_v2/views/search.py b/api_v2/views/search.py index b845871b..648a2f7f 100644 --- a/api_v2/views/search.py +++ b/api_v2/views/search.py @@ -40,8 +40,8 @@ def get_queryset(self): weighted_queryset = models.SearchResult.objects.raw( "SELECT 1 as id,rank, " + - "snippet(search_index,5,'','','...',20) as highlighted, " + - "* FROM search_index " + + "snippet(search_index,4,'','','...',20) as highlighted, " + + "document_pk,object_pk,object_name,object_model,text,schema_version FROM search_index " + "WHERE " + "schema_version LIKE %s " + "AND document_pk LIKE %s " + From dd0b14ac68d94f145649d3017d6b09e24d9c7542 Mon Sep 17 00:00:00 2001 From: August Johnson Date: Wed, 3 Apr 2024 20:31:59 -0500 Subject: [PATCH 04/14] Got the v2 objects loaded. --- api_v2/management/commands/index_v1.py | 11 +++++++++-- api_v2/models/background.py | 1 + api_v2/models/characterclass.py | 10 +++++++++- api_v2/models/creature.py | 9 +++++++++ api_v2/views/search.py | 2 +- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/api_v2/management/commands/index_v1.py b/api_v2/management/commands/index_v1.py index b2969683..753c7d14 100644 --- a/api_v2/management/commands/index_v1.py +++ b/api_v2/management/commands/index_v1.py @@ -38,7 +38,7 @@ def load_v1_content(self, model): def load_v2_content(self, model): results = [] - standard_v2_models = ['Item'] + standard_v2_models = ['Item','Spell','Creature','CharacterClass','Race','Feat','Condition','Background'] if model.__name__ in standard_v2_models: for o in model.objects.all(): @@ -48,7 +48,7 @@ def load_v2_content(self, model): object_name=o.name, object_model=o.__class__.__name__, schema_version='v2', - text=o.desc + text=o.as_text() )) return results @@ -120,6 +120,13 @@ def handle(self, *args, **options) -> None: self.load_content(v1.Armor,"v1") self.load_content(v2.Item,"v2") + self.load_content(v2.Spell,"v2") + self.load_content(v2.Creature,"v2") + self.load_content(v2.CharacterClass,"v2") + self.load_content(v2.Race,"v2") + self.load_content(v2.Feat,"v2") + self.load_content(v2.Condition,"v2") + self.load_content(v2.Background,"v2") # Take the content table's current data and load it into the index. diff --git a/api_v2/models/background.py b/api_v2/models/background.py index 7eb264f0..f263cc42 100644 --- a/api_v2/models/background.py +++ b/api_v2/models/background.py @@ -27,3 +27,4 @@ class Meta: """To assist with the UI layer.""" verbose_name_plural = "backgrounds" + diff --git a/api_v2/models/characterclass.py b/api_v2/models/characterclass.py index b23e2733..9766c003 100644 --- a/api_v2/models/characterclass.py +++ b/api_v2/models/characterclass.py @@ -97,4 +97,12 @@ def __str__(self): if self.is_subclass: return "{} [{}]".format(self.subclass_of.name, self.name) else: - return self.name \ No newline at end of file + return self.name + + def as_text(self): + text = self.name + '\n' + + for feature in self.feature_set.all(): + text+='\n' + feature.as_text() + + return text \ No newline at end of file diff --git a/api_v2/models/creature.py b/api_v2/models/creature.py index bbc9a876..03749caa 100644 --- a/api_v2/models/creature.py +++ b/api_v2/models/creature.py @@ -57,6 +57,15 @@ class Creature(Object, Abilities, FromDocument): max_length=100, help_text='The creature\'s allowed alignments.' ) + + def as_text(self): + text = self.name + '\n' + + for action in self.creatureaction_set.all(): + text+='\n' + action.as_text() + + return text + class CreatureAction(HasName, HasDescription, FromDocument): diff --git a/api_v2/views/search.py b/api_v2/views/search.py index 648a2f7f..fa61adfa 100644 --- a/api_v2/views/search.py +++ b/api_v2/views/search.py @@ -47,7 +47,7 @@ def get_queryset(self): "AND document_pk LIKE %s " + "AND object_model LIKE %s " + "AND search_index MATCH %s" + - "AND rank MATCH 'bm25(1.0, 1.0, 1.0, 10.0)'"+ # This line results in a 10x weight to Name + "AND rank MATCH 'bm25(1.0, 1.0, 10.0)'"+ # This line results in a 10x weight to Name "ORDER BY rank",[schema_version, document_pk, object_model, query]) return weighted_queryset From e7ed86b1633127893fc8b14618aa6f04532d615e Mon Sep 17 00:00:00 2001 From: August Johnson Date: Thu, 4 Apr 2024 16:59:16 -0500 Subject: [PATCH 05/14] Deleting, and adding the inhereted as_text function. --- api_v2/management/commands/{index_v1.py => buildindex.py} | 0 api_v2/models/document.py | 3 +++ 2 files changed, 3 insertions(+) rename api_v2/management/commands/{index_v1.py => buildindex.py} (100%) diff --git a/api_v2/management/commands/index_v1.py b/api_v2/management/commands/buildindex.py similarity index 100% rename from api_v2/management/commands/index_v1.py rename to api_v2/management/commands/buildindex.py diff --git a/api_v2/models/document.py b/api_v2/models/document.py index 1257510a..e3e52a33 100644 --- a/api_v2/models/document.py +++ b/api_v2/models/document.py @@ -108,6 +108,9 @@ class FromDocument(models.Model): max_length=100, help_text="Unique key for the Item.") + def as_text(self): + return "{}\n\n{}".format(self.name, self.desc) + def get_absolute_url(self): return reverse(self.__name__, kwargs={"pk": self.pk}) From 685fc337edba721b0391fa3e5592e1161bca589d Mon Sep 17 00:00:00 2001 From: August Johnson Date: Sun, 7 Apr 2024 08:17:09 -0500 Subject: [PATCH 06/14] fixing search results. --- api_v2/models/creature.py | 14 ++++++++++++++ api_v2/models/item.py | 7 +++++++ api_v2/serializers/search.py | 22 +++++++++++++++------- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/api_v2/models/creature.py b/api_v2/models/creature.py index 03749caa..4b778c22 100644 --- a/api_v2/models/creature.py +++ b/api_v2/models/creature.py @@ -66,6 +66,20 @@ def as_text(self): return text + def search_result_extra_fields(self): + return { + "armor_class":self.armor_class, + "hit_points":self.hit_points, + "hit_dice":self.hit_dice, + "strength":self.strength, + "dexterity":self.dexterity, + "constitution":self.constitution, + "intelligence":self.intelligence, + "wisdom":self.wisdom, + "charisma":self.charisma, + "challenge_rating":self.challenge_rating, + "cr":self.cr + } class CreatureAction(HasName, HasDescription, FromDocument): diff --git a/api_v2/models/item.py b/api_v2/models/item.py index 04e557b0..e748eedc 100644 --- a/api_v2/models/item.py +++ b/api_v2/models/item.py @@ -70,6 +70,13 @@ class Item(Object, HasDescription, FromDocument): def is_magic_item(self): return self.rarity is not None + def search_result_extra_fields(self): + return { + "type":self.category, + "rarity":self.rarity, + } + + class ItemSet(HasName, HasDescription, FromDocument): """A set of items to be referenced.""" diff --git a/api_v2/serializers/search.py b/api_v2/serializers/search.py index c189cd11..524df6fb 100644 --- a/api_v2/serializers/search.py +++ b/api_v2/serializers/search.py @@ -24,22 +24,30 @@ class Meta: 'highlighted'] def get_object(self, obj): + result_detail = None + if obj.schema_version == 'v1': if obj.object_model == 'MagicItem': result_detail = v1.MagicItem.objects.get(slug=obj.object_pk) - return result_detail.search_result_extra_fields() - if obj.object_model == 'Monster': result_detail = v1.Monster.objects.get(slug=obj.object_pk) - return result_detail.search_result_extra_fields() - if obj.object_model == 'Spell': result_detail = v1.Spell.objects.get(slug=obj.object_pk) - return result_detail.search_result_extra_fields() - if obj.object_model == 'Section': result_detail = v1.Section.objects.get(slug=obj.object_pk) - return result_detail.search_result_extra_fields() + + if obj.schema_version == 'v2': + if obj.object_model == 'Item': + result_detail = models.Item.objects.get(pk=obj.object_pk) + if obj.object_model == 'Creature': + result_detail = models.Creature.objects.get(pk=obj.object_pk) + if obj.object_model == 'Spell': + result_detail = models.Spell.objects.get(pk=obj.object_pk) + + if result_detail is not None: + return result_detail.search_result_extra_fields() + else: + return None def get_document(self, obj): if obj.schema_version == 'v1': From dfdd1ab9b3eb757d9d7c66b877a632db8eccfebe Mon Sep 17 00:00:00 2001 From: August Johnson Date: Tue, 2 Apr 2024 19:33:07 -0500 Subject: [PATCH 07/14] Adjusted quite a bit. --- api_v2/management/commands/index_v1.py | 7 +++--- api_v2/migrations/0073_auto_20240403_0010.py | 22 ++++++++++++++++++ api_v2/models/search.py | 23 +------------------ api_v2/serializers/search.py | 24 +++++++++++++------- api_v2/views/search.py | 17 +++++++------- 5 files changed, 51 insertions(+), 42 deletions(-) create mode 100644 api_v2/migrations/0073_auto_20240403_0010.py diff --git a/api_v2/management/commands/index_v1.py b/api_v2/management/commands/index_v1.py index 9054430c..9d1ca3a5 100644 --- a/api_v2/management/commands/index_v1.py +++ b/api_v2/management/commands/index_v1.py @@ -35,10 +35,9 @@ def load_content(self,model,schema): for o in model.objects.all(): search_results.append(v2.SearchResult( document_pk=o.document.slug, - document_name=o.document.title, object_pk=o.slug, object_name=o.name, - object_route=o.route, + object_model=o.__class__.__name__, schema_version="v1", text=o.name+"\n"+o.desc @@ -52,9 +51,9 @@ def load_index(self): cursor.execute("DROP TABLE IF EXISTS search_index;") - cursor.execute("CREATE VIRTUAL TABLE search_index USING FTS5(document_pk,document_name,object_pk,object_name,object_route,text,schema_version);") + cursor.execute("CREATE VIRTUAL TABLE search_index USING FTS5(document_pk,object_pk,object_name,object_model,text,schema_version);") - cursor.execute("INSERT INTO search_index (document_pk,document_name,object_pk,object_name,object_route,text,schema_version) SELECT document_pk,document_name,object_pk,object_name,object_route,text,schema_version FROM api_v2_searchresult") + cursor.execute("INSERT INTO search_index (document_pk,object_pk,object_name,object_model,text,schema_version) SELECT document_pk,object_pk,object_name,object_model,text,schema_version FROM api_v2_searchresult") def check_fts_enabled(self): diff --git a/api_v2/migrations/0073_auto_20240403_0010.py b/api_v2/migrations/0073_auto_20240403_0010.py new file mode 100644 index 00000000..a7298aa1 --- /dev/null +++ b/api_v2/migrations/0073_auto_20240403_0010.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.20 on 2024-04-03 00:10 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api_v2', '0072_merge_20240329_1829'), + ] + + operations = [ + migrations.RenameField( + model_name='searchresult', + old_name='object_route', + new_name='object_model', + ), + migrations.RemoveField( + model_name='searchresult', + name='document_name', + ), + ] diff --git a/api_v2/models/search.py b/api_v2/models/search.py index e2b7289b..7288a302 100644 --- a/api_v2/models/search.py +++ b/api_v2/models/search.py @@ -5,32 +5,11 @@ class SearchResult(models.Model): """ The Search Result object model""" document_pk = models.CharField(max_length=255) - document_name = models.CharField(max_length=100) object_pk = models.CharField(max_length=255) object_name = models.CharField(max_length=100) - object_route = models.CharField(max_length=255) + object_model = models.CharField(max_length=255) schema_version = models.CharField(max_length=100) rank = models.DecimalField(max_digits=10, decimal_places=4, null=True, default=None) text = models.TextField(null=True, default=None) highlighted = models.TextField(null=True, default=None) - - @property - def document_slug(self): - return self.document_pk - - @property - def document_title(self): - return self.document_name - - @property - def route(self): - return self.object_route - - @property - def slug(self): - return self.object_pk - - @property - def name(self): - return self.object_name diff --git a/api_v2/serializers/search.py b/api_v2/serializers/search.py index 16ba1bd2..8fa8f7e2 100644 --- a/api_v2/serializers/search.py +++ b/api_v2/serializers/search.py @@ -10,17 +10,17 @@ class SearchResultSerializer(serializers.ModelSerializer): text = serializers.ReadOnlyField() highlighted = serializers.ReadOnlyField() object = serializers.SerializerMethodField(method_name='get_object') + document = serializers.SerializerMethodField(method_name='get_document') class Meta: model = models.SearchResult fields = [ - 'document_pk', - 'document_name', + 'document', 'object_pk', 'object_name', 'object', - 'object_route', + 'object_model', 'schema_version', 'rank', 'text', @@ -28,18 +28,26 @@ class Meta: def get_object(self, obj): if obj.schema_version == 'v1': - if obj.object_route == 'magicitems/': + if obj.object_model == 'MagicItem': result_detail = v1.MagicItem.objects.get(slug=obj.object_pk) return result_detail.search_result_extra_fields() - if obj.object_route == 'monsters/': + if obj.object_model == 'Monster': result_detail = v1.Monster.objects.get(slug=obj.object_pk) return result_detail.search_result_extra_fields() - if obj.object_route == 'spells/': + if obj.object_model == 'Spell': result_detail = v1.Spell.objects.get(slug=obj.object_pk) return result_detail.search_result_extra_fields() - if obj.object_route == 'sections/': + if obj.object_model == 'Section': result_detail = v1.Section.objects.get(slug=obj.object_pk) - return result_detail.search_result_extra_fields() \ No newline at end of file + return result_detail.search_result_extra_fields() + + def get_document(self, obj): + if obj.schema_version == 'v1': + doc = v1.Document.objects.get(slug=obj.document_pk) + return { + 'document_pk': doc.slug, + 'document_name': doc.title + } diff --git a/api_v2/views/search.py b/api_v2/views/search.py index f7510d84..b845871b 100644 --- a/api_v2/views/search.py +++ b/api_v2/views/search.py @@ -33,20 +33,21 @@ def get_queryset(self): else: document_pk = self.request.query_params.get("document_pk") - if self.request.query_params.get("object_route") is None: - object_route = '%' + if self.request.query_params.get("object_model") is None: + object_model = '%' else: - object_route = self.request.query_params.get("object_route") + object_model = self.request.query_params.get("object_model") - queryset = models.SearchResult.objects.raw( + weighted_queryset = models.SearchResult.objects.raw( "SELECT 1 as id,rank, " + "snippet(search_index,5,'','','...',20) as highlighted, " + "* FROM search_index " + "WHERE " + "schema_version LIKE %s " + "AND document_pk LIKE %s " + - "AND object_route LIKE %s " + - "AND text MATCH %s " + - "ORDER BY rank",[schema_version, document_pk, object_route, query]) + "AND object_model LIKE %s " + + "AND search_index MATCH %s" + + "AND rank MATCH 'bm25(1.0, 1.0, 1.0, 10.0)'"+ # This line results in a 10x weight to Name + "ORDER BY rank",[schema_version, document_pk, object_model, query]) - return queryset + return weighted_queryset From c62cfcb4a6281979928e8188e520ab6a055b8c50 Mon Sep 17 00:00:00 2001 From: August Johnson Date: Tue, 2 Apr 2024 19:51:45 -0500 Subject: [PATCH 08/14] First v2 model being indexed. --- api_v2/management/commands/index_v1.py | 69 +++++++++++++++++++------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/api_v2/management/commands/index_v1.py b/api_v2/management/commands/index_v1.py index 9d1ca3a5..ba3e8381 100644 --- a/api_v2/management/commands/index_v1.py +++ b/api_v2/management/commands/index_v1.py @@ -18,22 +18,14 @@ def unload_all_content(self): v2.SearchResult.objects.all().delete() print("UNLOADED_OBJECT_COUNT:{}".format(object_count)) - - def load_content(self,model,schema): - print("SCHEMA:{} OBJECT_COUNT:{} MODEL:{} TABLE_NAME:{}".format( - schema, - model.objects.all().count(), - model.__name__, - model._meta.db_table)) - + def load_v1_content(self, model): + results = [] standard_v1_models = ['MagicItem','Spell','Monster','CharClass','Archetype', - 'Race','Subrace','Plane','Section','Feat','Condition','Background','Weapon','Armor'] - - search_results = [] + 'Race','Subrace','Plane','Section','Feat','Condition','Background','Weapon','Armor'] - if model.__name__ in standard_v1_models and schema=='v1': + if model.__name__ in standard_v1_models: for o in model.objects.all(): - search_results.append(v2.SearchResult( + results.append(v2.SearchResult( document_pk=o.document.slug, object_pk=o.slug, object_name=o.name, @@ -42,19 +34,58 @@ def load_content(self,model,schema): text=o.name+"\n"+o.desc )) + return results + + + def load_v2_content(self, model): + results = [] + standard_v2_models = ['Item'] + + if model.__name__ in standard_v2_models: + for o in model.objects.all(): + results.append(v2.SearchResult( + document_pk=o.document.pk, + object_pk=o.pk, + object_name=o.name, + object_model=o.__class__.__name__, + schema_version='v2', + text=o.desc + )) + return results + + + def load_content(self,model,schema): + print("SCHEMA:{} OBJECT_COUNT:{} MODEL:{} TABLE_NAME:{}".format( + schema, + model.objects.all().count(), + model.__name__, + model._meta.db_table)) - v2.SearchResult.objects.bulk_create(search_results) + if schema == 'v1': + v2.SearchResult.objects.bulk_create( + self.load_v1_content(model) + ) + if schema == 'v2': + v2.SearchResult.objects.bulk_create( + self.load_v2_content(model) + ) def load_index(self): with connection.cursor() as cursor: cursor.execute("DROP TABLE IF EXISTS search_index;") - - cursor.execute("CREATE VIRTUAL TABLE search_index USING FTS5(document_pk,object_pk,object_name,object_model,text,schema_version);") - cursor.execute("INSERT INTO search_index (document_pk,object_pk,object_name,object_model,text,schema_version) SELECT document_pk,object_pk,object_name,object_model,text,schema_version FROM api_v2_searchresult") - + cursor.execute( + "CREATE VIRTUAL TABLE search_index " + + "USING FTS5(document_pk,object_pk,object_name,object_model,text,schema_version);") + + cursor.execute( + "INSERT INTO search_index " + + "(document_pk,object_pk,object_name,object_model,text,schema_version) " + + "SELECT document_pk,object_pk,object_name,object_model,text,schema_version " + + "FROM api_v2_searchresult") + def check_fts_enabled(self): #import sqlite3 @@ -90,6 +121,8 @@ def handle(self, *args, **options) -> None: self.load_content(v1.Weapon,"v1") self.load_content(v1.Armor,"v1") + self.load_content(v2.Item,"v2") + # Take the content table's current data and load it into the index. self.load_index() From 4c1480c2d5e7a7531f828e081e8da81349ca08d6 Mon Sep 17 00:00:00 2001 From: August Johnson Date: Wed, 3 Apr 2024 18:15:22 -0500 Subject: [PATCH 09/14] some basic refactors. --- api_v2/management/commands/index_v1.py | 4 +--- api_v2/serializers/search.py | 14 +++++++++----- api_v2/views/search.py | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/api_v2/management/commands/index_v1.py b/api_v2/management/commands/index_v1.py index ba3e8381..b2969683 100644 --- a/api_v2/management/commands/index_v1.py +++ b/api_v2/management/commands/index_v1.py @@ -36,7 +36,6 @@ def load_v1_content(self, model): )) return results - def load_v2_content(self, model): results = [] standard_v2_models = ['Item'] @@ -44,7 +43,7 @@ def load_v2_content(self, model): if model.__name__ in standard_v2_models: for o in model.objects.all(): results.append(v2.SearchResult( - document_pk=o.document.pk, + document_pk=o.document.key, object_pk=o.pk, object_name=o.name, object_model=o.__class__.__name__, @@ -53,7 +52,6 @@ def load_v2_content(self, model): )) return results - def load_content(self,model,schema): print("SCHEMA:{} OBJECT_COUNT:{} MODEL:{} TABLE_NAME:{}".format( schema, diff --git a/api_v2/serializers/search.py b/api_v2/serializers/search.py index 8fa8f7e2..c189cd11 100644 --- a/api_v2/serializers/search.py +++ b/api_v2/serializers/search.py @@ -6,9 +6,6 @@ from api import models as v1 class SearchResultSerializer(serializers.ModelSerializer): - rank = serializers.ReadOnlyField() - text = serializers.ReadOnlyField() - highlighted = serializers.ReadOnlyField() object = serializers.SerializerMethodField(method_name='get_object') document = serializers.SerializerMethodField(method_name='get_document') @@ -48,6 +45,13 @@ def get_document(self, obj): if obj.schema_version == 'v1': doc = v1.Document.objects.get(slug=obj.document_pk) return { - 'document_pk': doc.slug, - 'document_name': doc.title + 'key': doc.slug, + 'name': doc.title } + + if obj.schema_version == 'v2': + doc = models.Document.objects.get(key=obj.document_pk) + return { + 'key': doc.key, + 'name': doc.name + } \ No newline at end of file diff --git a/api_v2/views/search.py b/api_v2/views/search.py index b845871b..648a2f7f 100644 --- a/api_v2/views/search.py +++ b/api_v2/views/search.py @@ -40,8 +40,8 @@ def get_queryset(self): weighted_queryset = models.SearchResult.objects.raw( "SELECT 1 as id,rank, " + - "snippet(search_index,5,'','','...',20) as highlighted, " + - "* FROM search_index " + + "snippet(search_index,4,'','','...',20) as highlighted, " + + "document_pk,object_pk,object_name,object_model,text,schema_version FROM search_index " + "WHERE " + "schema_version LIKE %s " + "AND document_pk LIKE %s " + From 09ecf5956cf2ec92c966e93fa1ee920674902465 Mon Sep 17 00:00:00 2001 From: August Johnson Date: Wed, 3 Apr 2024 20:31:59 -0500 Subject: [PATCH 10/14] Got the v2 objects loaded. --- api_v2/management/commands/index_v1.py | 11 +++++++++-- api_v2/models/background.py | 1 + api_v2/models/characterclass.py | 10 +++++++++- api_v2/models/creature.py | 9 +++++++++ api_v2/views/search.py | 2 +- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/api_v2/management/commands/index_v1.py b/api_v2/management/commands/index_v1.py index b2969683..753c7d14 100644 --- a/api_v2/management/commands/index_v1.py +++ b/api_v2/management/commands/index_v1.py @@ -38,7 +38,7 @@ def load_v1_content(self, model): def load_v2_content(self, model): results = [] - standard_v2_models = ['Item'] + standard_v2_models = ['Item','Spell','Creature','CharacterClass','Race','Feat','Condition','Background'] if model.__name__ in standard_v2_models: for o in model.objects.all(): @@ -48,7 +48,7 @@ def load_v2_content(self, model): object_name=o.name, object_model=o.__class__.__name__, schema_version='v2', - text=o.desc + text=o.as_text() )) return results @@ -120,6 +120,13 @@ def handle(self, *args, **options) -> None: self.load_content(v1.Armor,"v1") self.load_content(v2.Item,"v2") + self.load_content(v2.Spell,"v2") + self.load_content(v2.Creature,"v2") + self.load_content(v2.CharacterClass,"v2") + self.load_content(v2.Race,"v2") + self.load_content(v2.Feat,"v2") + self.load_content(v2.Condition,"v2") + self.load_content(v2.Background,"v2") # Take the content table's current data and load it into the index. diff --git a/api_v2/models/background.py b/api_v2/models/background.py index 7eb264f0..f263cc42 100644 --- a/api_v2/models/background.py +++ b/api_v2/models/background.py @@ -27,3 +27,4 @@ class Meta: """To assist with the UI layer.""" verbose_name_plural = "backgrounds" + diff --git a/api_v2/models/characterclass.py b/api_v2/models/characterclass.py index b23e2733..9766c003 100644 --- a/api_v2/models/characterclass.py +++ b/api_v2/models/characterclass.py @@ -97,4 +97,12 @@ def __str__(self): if self.is_subclass: return "{} [{}]".format(self.subclass_of.name, self.name) else: - return self.name \ No newline at end of file + return self.name + + def as_text(self): + text = self.name + '\n' + + for feature in self.feature_set.all(): + text+='\n' + feature.as_text() + + return text \ No newline at end of file diff --git a/api_v2/models/creature.py b/api_v2/models/creature.py index a2e6b94a..f5657382 100644 --- a/api_v2/models/creature.py +++ b/api_v2/models/creature.py @@ -57,6 +57,15 @@ class Creature(Object, Abilities, FromDocument): max_length=100, help_text='The creature\'s allowed alignments.' ) + + def as_text(self): + text = self.name + '\n' + + for action in self.creatureaction_set.all(): + text+='\n' + action.as_text() + + return text + @property def creatureset(self): diff --git a/api_v2/views/search.py b/api_v2/views/search.py index 648a2f7f..fa61adfa 100644 --- a/api_v2/views/search.py +++ b/api_v2/views/search.py @@ -47,7 +47,7 @@ def get_queryset(self): "AND document_pk LIKE %s " + "AND object_model LIKE %s " + "AND search_index MATCH %s" + - "AND rank MATCH 'bm25(1.0, 1.0, 1.0, 10.0)'"+ # This line results in a 10x weight to Name + "AND rank MATCH 'bm25(1.0, 1.0, 10.0)'"+ # This line results in a 10x weight to Name "ORDER BY rank",[schema_version, document_pk, object_model, query]) return weighted_queryset From 0a0679217c962df2f1c6d2335afbb87ce8a37c95 Mon Sep 17 00:00:00 2001 From: August Johnson Date: Thu, 4 Apr 2024 16:59:16 -0500 Subject: [PATCH 11/14] Deleting, and adding the inhereted as_text function. --- api_v2/management/commands/{index_v1.py => buildindex.py} | 0 api_v2/models/document.py | 3 +++ 2 files changed, 3 insertions(+) rename api_v2/management/commands/{index_v1.py => buildindex.py} (100%) diff --git a/api_v2/management/commands/index_v1.py b/api_v2/management/commands/buildindex.py similarity index 100% rename from api_v2/management/commands/index_v1.py rename to api_v2/management/commands/buildindex.py diff --git a/api_v2/models/document.py b/api_v2/models/document.py index dbd7e551..f8e387b7 100644 --- a/api_v2/models/document.py +++ b/api_v2/models/document.py @@ -112,6 +112,9 @@ class FromDocument(models.Model): max_length=100, help_text="Unique key for the Item.") + def as_text(self): + return "{}\n\n{}".format(self.name, self.desc) + def get_absolute_url(self): return reverse(self.__name__, kwargs={"pk": self.pk}) From 980096b3c504f4fd8a402e10e1c28ee03fc13328 Mon Sep 17 00:00:00 2001 From: August Johnson Date: Sun, 7 Apr 2024 08:17:09 -0500 Subject: [PATCH 12/14] fixing search results. --- api_v2/models/creature.py | 14 ++++++++++++++ api_v2/models/item.py | 7 +++++++ api_v2/serializers/search.py | 22 +++++++++++++++------- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/api_v2/models/creature.py b/api_v2/models/creature.py index f5657382..c81a2543 100644 --- a/api_v2/models/creature.py +++ b/api_v2/models/creature.py @@ -66,6 +66,20 @@ def as_text(self): return text + def search_result_extra_fields(self): + return { + "armor_class":self.armor_class, + "hit_points":self.hit_points, + "hit_dice":self.hit_dice, + "strength":self.strength, + "dexterity":self.dexterity, + "constitution":self.constitution, + "intelligence":self.intelligence, + "wisdom":self.wisdom, + "charisma":self.charisma, + "challenge_rating":self.challenge_rating, + "cr":self.cr + } @property def creatureset(self): diff --git a/api_v2/models/item.py b/api_v2/models/item.py index 04e557b0..e748eedc 100644 --- a/api_v2/models/item.py +++ b/api_v2/models/item.py @@ -70,6 +70,13 @@ class Item(Object, HasDescription, FromDocument): def is_magic_item(self): return self.rarity is not None + def search_result_extra_fields(self): + return { + "type":self.category, + "rarity":self.rarity, + } + + class ItemSet(HasName, HasDescription, FromDocument): """A set of items to be referenced.""" diff --git a/api_v2/serializers/search.py b/api_v2/serializers/search.py index c189cd11..524df6fb 100644 --- a/api_v2/serializers/search.py +++ b/api_v2/serializers/search.py @@ -24,22 +24,30 @@ class Meta: 'highlighted'] def get_object(self, obj): + result_detail = None + if obj.schema_version == 'v1': if obj.object_model == 'MagicItem': result_detail = v1.MagicItem.objects.get(slug=obj.object_pk) - return result_detail.search_result_extra_fields() - if obj.object_model == 'Monster': result_detail = v1.Monster.objects.get(slug=obj.object_pk) - return result_detail.search_result_extra_fields() - if obj.object_model == 'Spell': result_detail = v1.Spell.objects.get(slug=obj.object_pk) - return result_detail.search_result_extra_fields() - if obj.object_model == 'Section': result_detail = v1.Section.objects.get(slug=obj.object_pk) - return result_detail.search_result_extra_fields() + + if obj.schema_version == 'v2': + if obj.object_model == 'Item': + result_detail = models.Item.objects.get(pk=obj.object_pk) + if obj.object_model == 'Creature': + result_detail = models.Creature.objects.get(pk=obj.object_pk) + if obj.object_model == 'Spell': + result_detail = models.Spell.objects.get(pk=obj.object_pk) + + if result_detail is not None: + return result_detail.search_result_extra_fields() + else: + return None def get_document(self, obj): if obj.schema_version == 'v1': From 404e710419dc1ce42dab2d70372b9899a4848af5 Mon Sep 17 00:00:00 2001 From: August Johnson Date: Tue, 9 Apr 2024 07:16:58 -0500 Subject: [PATCH 13/14] getting rarity exposed reliably. --- api_v2/models/creature.py | 10 +--------- api_v2/models/item.py | 8 ++++---- api_v2/serializers/search.py | 1 + 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/api_v2/models/creature.py b/api_v2/models/creature.py index c81a2543..92e72fbb 100644 --- a/api_v2/models/creature.py +++ b/api_v2/models/creature.py @@ -70,15 +70,7 @@ def search_result_extra_fields(self): return { "armor_class":self.armor_class, "hit_points":self.hit_points, - "hit_dice":self.hit_dice, - "strength":self.strength, - "dexterity":self.dexterity, - "constitution":self.constitution, - "intelligence":self.intelligence, - "wisdom":self.wisdom, - "charisma":self.charisma, - "challenge_rating":self.challenge_rating, - "cr":self.cr + "ability_scores":self.get_ability_scores(), } @property diff --git a/api_v2/models/item.py b/api_v2/models/item.py index e748eedc..07f15ba0 100644 --- a/api_v2/models/item.py +++ b/api_v2/models/item.py @@ -71,10 +71,10 @@ def is_magic_item(self): return self.rarity is not None def search_result_extra_fields(self): - return { - "type":self.category, - "rarity":self.rarity, - } + fields = {"type":self.category.key} + if self.is_magic_item: + fields["rarity"]=self.rarity.key + return fields diff --git a/api_v2/serializers/search.py b/api_v2/serializers/search.py index 524df6fb..76d2fd4a 100644 --- a/api_v2/serializers/search.py +++ b/api_v2/serializers/search.py @@ -4,6 +4,7 @@ from api_v2 import models from api import models as v1 +from .item import ItemSerializer class SearchResultSerializer(serializers.ModelSerializer): object = serializers.SerializerMethodField(method_name='get_object') From 002dd467b0dccb36b46c1c91aaa731a2e83397f0 Mon Sep 17 00:00:00 2001 From: August Johnson Date: Tue, 9 Apr 2024 08:44:54 -0500 Subject: [PATCH 14/14] Adding reasonable route. --- api_v2/models/document.py | 5 +++++ api_v2/serializers/search.py | 26 ++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/api_v2/models/document.py b/api_v2/models/document.py index f8e387b7..c04dd710 100644 --- a/api_v2/models/document.py +++ b/api_v2/models/document.py @@ -118,5 +118,10 @@ def as_text(self): def get_absolute_url(self): return reverse(self.__name__, kwargs={"pk": self.pk}) + def search_result_extra_fields(self): + return { + "school":self.school.key, + } + class Meta: abstract = True \ No newline at end of file diff --git a/api_v2/serializers/search.py b/api_v2/serializers/search.py index 76d2fd4a..1ad93084 100644 --- a/api_v2/serializers/search.py +++ b/api_v2/serializers/search.py @@ -4,11 +4,12 @@ from api_v2 import models from api import models as v1 -from .item import ItemSerializer +from django.urls import reverse class SearchResultSerializer(serializers.ModelSerializer): object = serializers.SerializerMethodField(method_name='get_object') document = serializers.SerializerMethodField(method_name='get_document') + route = serializers.SerializerMethodField(method_name='get_route') class Meta: @@ -20,6 +21,7 @@ class Meta: 'object', 'object_model', 'schema_version', + 'route', 'rank', 'text', 'highlighted'] @@ -63,4 +65,24 @@ def get_document(self, obj): return { 'key': doc.key, 'name': doc.name - } \ No newline at end of file + } + + def get_route(self, obj): + # May want to split this out into v1 and v2? + route_lookup = { + "Item":"items", + "Creature":"creatures", + "Spell":"spells", + "CharacterClass":"class", + "Monster":"monsters", + "MagicItem":"magicitems", + "Section":"sections", + "Background":"backgrounds", + "Subrace":"subraces", + "Feat":"feats", + "Race":"races", + "Plane":"planes", + } + + route = "{}/{}/".format(obj.schema_version,route_lookup[obj.object_model]) + return route \ No newline at end of file