Merge pull request #62 from Chia-Network/dev/michael.taylor/fts-fixes

fix: fts fixes so that they are index correctly

README.md

@@ -96,7 +96,9 @@ To run a FTS query on a supported table, you can use the `MATCH` operator. A vir
 Example:
 
 ```
-SELECT * FROM projects WHERE projectName MATCH "partial name" ORDER BY rank
+SELECT rank, * FROM projects_fts WHERE projects_fts MATCH "PartialMatch*" ORDER BY rank
+
+The '*' in the match is needed for wildcard
 ```
 
-More info: https://www.sqlite.org/fts5.html
+More info: https://www.sqlite.org/fts5.html

migrations/20211212200953-fulltext-search.cjs

@@ -4,10 +4,50 @@ module.exports = {
   up: async (queryInterface, Sequelize) => {
     if (queryInterface.sequelize.getDialect() === 'sqlite') {
       await queryInterface.sequelize.query(`
-      CREATE VIRTUAL TABLE projects_fts USING fts5(projects);
+      CREATE VIRTUAL TABLE projects_fts USING fts5(
+        id,
+        warehouseProjectId,
+        projectId,
+        projectLocationId,
+        currentRegistry,
+        registryOfOrigin,
+        originProjectId,
+        program,
+        projectName,
+        projectLink,
+        projectDeveloper,
+        sector,
+        projectType,
+        coveredByNDC,
+        NDCLinkage,
+        projectStatus,
+        projectStatusDate,
+        unitMetric,
+        methodology,
+        methodologyVersion,
+        validationApproach,
+        validationDate,
+        projectTag,
+        estimatedAnnualAverageEmissionReduction
+      );
       `);
       await queryInterface.sequelize.query(`
-      CREATE VIRTUAL TABLE units_fts USING fts5(units);
+      CREATE VIRTUAL TABLE units_fts USING fts5(
+          id,
+          buyer,
+          registry,
+          blockIdentifier,
+          identifier,
+          unitType,
+          unitCount,
+          unitStatus,
+          unitStatusDate,
+          transactionType,
+          unitIssuanceLocation,
+          unitLink,
+          correspondingAdjustment,
+          unitTag
+        );
       `);
     }
   },

migrations/20211219182106-sqlite-triggers-projects.cjs

@@ -108,13 +108,14 @@ module.exports = {
         );
       END;
       `);
-    }},
+    }
+  },
 
   down: async (queryInterface, Sequelize) => {
     if (queryInterface.sequelize.getDialect() === 'sqlite') {
-      await queryInterface.sequelize.query("DROP TRIGGER project_insert_fts;");
-      await queryInterface.sequelize.query("DROP TRIGGER project_delete_fts;");
-      await queryInterface.sequelize.query("DROP TRIGGER project_update_fts;");
+      await queryInterface.sequelize.query('DROP TRIGGER project_insert_fts;');
+      await queryInterface.sequelize.query('DROP TRIGGER project_delete_fts;');
+      await queryInterface.sequelize.query('DROP TRIGGER project_update_fts;');
     }
-  }
+  },
 };

migrations/20211219184405-sqlite-triggers-units.cjs

@@ -7,12 +7,10 @@ module.exports = {
       CREATE TRIGGER unit_insert_fts AFTER INSERT ON units BEGIN
         INSERT INTO units_fts(
           id,
-          projectId, 
           buyer, 
           registry, 
           blockIdentifier, 
           identifier,
-          qualificationId, 
           unitType, 
           unitCount, 
           unitStatus, 
@@ -21,16 +19,13 @@ module.exports = {
           unitIssuanceLocation,
           unitLink, 
           correspondingAdjustment,
-          unitTag, 
-          vintageId
+          unitTag
         ) VALUES (
           new.id,
-          new.projectId, 
           new.buyer, 
           new.registry, 
           new.blockIdentifier, 
           new.identifier,
-          new.qualificationId, 
           new.unitType, 
           new.unitCount, 
           new.unitStatus, 
@@ -39,8 +34,7 @@ module.exports = {
           new.unitIssuanceLocation,
           new.unitLink, 
           new.correspondingAdjustment,
-          new.unitTag, 
-          new.vintageId
+          new.unitTag
         );
       END;`);
 
@@ -97,9 +91,9 @@ module.exports = {
 
   down: async (queryInterface, Sequelize) => {
     if (queryInterface.sequelize.getDialect() === 'sqlite') {
-      await queryInterface.sequelize.query("DROP TRIGGER unit_insert_fts;");
-      await queryInterface.sequelize.query("DROP TRIGGER unit_delete_fts;");
-      await queryInterface.sequelize.query("DROP TRIGGER unit_update_fts;");
+      await queryInterface.sequelize.query('DROP TRIGGER unit_insert_fts;');
+      await queryInterface.sequelize.query('DROP TRIGGER unit_delete_fts;');
+      await queryInterface.sequelize.query('DROP TRIGGER unit_update_fts;');
     }
-  }
+  },
 };

src/controllers/staging.controller.js

@@ -1,5 +1,6 @@
 import _ from 'lodash';
 import * as fullNode from '../fullnode';
+
 import { Staging, StagingMock, Project, Unit } from '../models';
 
 export const findAll = async (req, res) => {
@@ -52,9 +53,11 @@ export const findAll = async (req, res) => {
 };
 
 export const commit = async (req, res) => {
-  const queryResponse = await Staging.findAll();
-  const stagingRecords = queryResponse.dataValues;
-  stagingRecords.forEach(async (stagingRecord) => {
+  const queryResponses = await Staging.findAll();
+
+  queryResponses.forEach(async (queryResponse) => {
+    const stagingRecord = queryResponse.dataValues;
+
     const {
       id: stagingRecordId,
       uuid,

src/fullnode/simulator.js

@@ -4,16 +4,26 @@ const THIRTY_SEC = 30000;
 // Simulate 30 seconds passing before commited to node
 
 export const updateProjectRecord = async (uuid, record, stagingRecordId) => {
-  await deleteProjectRecord(uuid);
-  await createProjectRecord(uuid, record);
-
-  if (stagingRecordId) {
-    await Staging.destroy({
-      where: {
-        id: stagingRecordId,
-      },
-    });
-  }
+  return new Promise((resolve) => {
+    setTimeout(async () => {
+      if (stagingRecordId) {
+        await Project.destroy({
+          where: {
+            warehouseProjectId: uuid,
+          },
+        });
+        await Project.create({
+          ...record,
+          warehouseProjectId: uuid,
+        });
+        await Staging.destroy({
+          where: {
+            id: stagingRecordId,
+          },
+        });
+      }
+    }, THIRTY_SEC);
+  });
 };
 
 export const createProjectRecord = (uuid, record, stagingRecordId) => {

src/models/projects/projects.model.js

@@ -39,29 +39,11 @@ class Project extends Model {
   }
 
   static findAllSqliteFts(queryStr) {
-    const sql = `SELECT * FROM projects WHERE 
-                    warehouseProjectId MATCH ":search" OR
-                    currentRegistry MATCH ":search" OR
-                    registryOfOrigin MATCH ":search" OR
-                    program MATCH ":search" OR
-                    projectName MATCH ":search" OR
-                    projectLink MATCH ":search" OR
-                    projectDeveloper MATCH ":search" OR
-                    sector MATCH ":search" OR
-                    projectType MATCH ":search" OR
-                    NDCLinkage MATCH ":search" OR
-                    projectStatus MATCH ":search" OR
-                    unitMetric MATCH ":search" OR
-                    methodology MATCH ":search" OR
-                    methodologyVersion MATCH ":search" OR
-                    validationApproach MATCH ":search" OR
-                    projectTag MATCH ":search" OR
-                    estimatedAnnualAverageEmissionReduction MATCH ":search" OR
-                    ORDER BY rank DESC`;
+    const sql = `SELECT * FROM projects_fts WHERE projects_fts MATCH :search ORDER BY rank DESC`;
 
     return sequelize.query(sql, {
       model: Project,
-      replacements: { search: queryStr },
+      replacements: { search: `${queryStr}*` },
       mapToModel: true, // pass true here if you have any mapped fields
     });
   }

src/models/projects/projects.modeltypes.cjs

@@ -7,7 +7,7 @@ module.exports = {
     autoIncrement: true,
   },
   warehouseProjectId: Sequelize.STRING,
-  projectID: Sequelize.STRING,
+  projectId: Sequelize.STRING,
   projectLocationId: Sequelize.NUMBER,
   currentRegistry: Sequelize.STRING,
   registryOfOrigin: Sequelize.STRING,