Merge pull request #91 from jghauser/fix-search-precalc

Fix search precalc

.gitignore

@@ -1,5 +1,4 @@
-dev
-doc/tags
+/doc
 .devenv/
 .direnv/
 .envrc

doc/papis.txt

@@ -1,4 +1,4 @@
-*papis.txt*              For NVIM v0.8.0             Last change: 2024 July 08
+*papis.txt*              For NVIM v0.8.0             Last change: 2024 July 29
 
 ==============================================================================
 Table of Contents                                    *papis-table-of-contents*

lua/papis/commands.lua

@@ -55,7 +55,6 @@ end
 
 ---Creates the main Papis command
 local function create_command()
-  -- NOTE: the options will vary, based on your use case.
   vim.api.nvim_create_user_command("Papis", papis_cmd, {
     nargs = "+",
     desc = "Papis main command (with subcommands)",

lua/papis/config.lua

@@ -30,7 +30,7 @@ local default_config = {
   enable_keymaps = false,
   enable_fs_watcher = true,
   data_tbl_schema = { -- only "text" and "luatable" are allowed
-    id = { "integer", pk = true },
+    id = { "integer", primary = true },
     papis_id = { "text", required = true, unique = true },
     ref = { "text", required = true, unique = true },
     author = "text",

lua/papis/data.lua

@@ -65,6 +65,7 @@ local function update_main_tbls(metadata)
         log.debug("Changing an existing entry")
         id = id[1].id
         db.data:clean_update({ id = id }, data_row)
+        metadata_row.entry = id
         db.metadata:update({ id = id }, metadata_row)
       else
         log.debug("Adding a new entry")
@@ -138,6 +139,8 @@ function M.update_db(metadata)
   log.debug("Updating the database")
   update_main_tbls(metadata)
   update_module_tbls(metadata)
+  local db_last_modified = os.time()
+  db.state:update({ id = 1 }, { db_last_modified = db_last_modified })
 end
 
 ---Resets the database

lua/papis/fs-watcher.lua

@@ -9,7 +9,7 @@
 
 local Path = require("pathlib")
 
-local uv = vim.loop
+local uv = vim.uv
 local fs_stat = uv.fs_stat
 local new_timer = uv.new_timer
 local api = vim.api
@@ -27,6 +27,7 @@ local file_read_timer
 local autocmd_id
 local handles = {}
 local fs_watching_stopped = false
+local event_timestamps = {}
 
 ---Uses libuv to start file system watchers
 ---@param path string #The path to watch
@@ -78,55 +79,67 @@ local function init_fs_watcher(dir_to_watch, is_library_root)
     local info_path
     local do_unwatch = false
     local do_update = true
-    if is_library_root then
-      log.debug("Filesystem event in the library root directory")
-      entry_dir = Path(dir_to_watch, filename)
-      info_path = entry_dir / info_name
-      local entry_dir_str = tostring(entry_dir)
-      local info_path_str = tostring(info_path)
-      if entry_dir:exists() and entry_dir:is_dir() then
-        log.debug(string.format("Filesystem event: path '%s' added", entry_dir_str))
-        init_fs_watcher(entry_dir_str)
-        if info_path:exists() then
-          mtime = fs_stat(info_path_str).mtime.sec
+    local current_time = uv.hrtime() / 1e6 -- get current time in milliseconds
+    local last_event_time = event_timestamps[filename]
+
+    if last_event_time and current_time - last_event_time < 200 then
+      log.debug("Debouncing: skipping filesystem event")
+      -- If the last event for this file was less than 200ms ago, discard this event
+      return
+    end
+
+    -- Update the timestamp for this file
+    event_timestamps[filename] = current_time
+
+    vim.defer_fn(function()
+      if is_library_root then
+        log.debug("Filesystem event in the library root directory")
+        entry_dir = Path(dir_to_watch, filename)
+        info_path = entry_dir / info_name
+        local entry_dir_str = tostring(entry_dir)
+        local info_path_str = tostring(info_path)
+        if entry_dir:exists() and entry_dir:is_dir() then
+          log.debug(string.format("Filesystem event: path '%s' added", entry_dir_str))
+          init_fs_watcher(entry_dir_str)
+          if info_path:exists() then
+            mtime = fs_stat(info_path_str).mtime.sec
+          end
+        elseif entry_dir:is_dir() then
+          log.debug(string.format("Filesystem event: path '' removed", entry_dir_str))
+          -- don't update here, because we'll catch it below under entry events
+          do_update = false
+        else
+          -- it's a file (not a directory). ignore
+          do_update = false
         end
-      elseif entry_dir:is_dir() then
-        log.debug(string.format("Filesystem event: path '' removed", entry_dir_str))
-        -- don't update here, because we'll catch it below under entry events
-        do_update = false
       else
-        -- it's a file (not a directory). ignore
-        do_update = false
-      end
-    else
-      log.debug("Filesystem event in entry directory")
-      entry_dir = Path(dir_to_watch)
-      info_path = entry_dir / info_name
-      local info_path_str = tostring(info_path)
-      if info_path:exists() then
-        -- info file exists, update with new info
-        log.debug(string.format("Filesystem event: '%s' changed", info_path_str))
-        mtime = fs_stat(tostring(info_path)).mtime.sec
-      elseif not entry_dir:exists() then
-        -- info file and entry dir don't exist. delete entry (mtime = nil) and remove watcher
-        log.debug(string.format("Filesystem event: '%s' removed", info_path_str))
-        do_unwatch = true
-      else
-        -- info file doesn't exist but entry dir does. delete entry but keep watcher
-        log.debug(string.format("Filesystem event: '%s' removed", info_path_str))
+        log.debug("Filesystem event in entry directory")
+        entry_dir = Path(dir_to_watch)
+        info_path = entry_dir / info_name
+        local info_path_str = tostring(info_path)
+        if info_path:exists() then
+          -- info file exists, update with new info
+          log.debug(string.format("Filesystem event: '%s' changed", info_path_str))
+          mtime = fs_stat(tostring(info_path)).mtime.sec
+        elseif not entry_dir:exists() then
+          -- info file and entry dir don't exist. delete entry (mtime = nil) and remove watcher
+          log.debug(string.format("Filesystem event: '%s' removed", info_path_str))
+          do_unwatch = true
+        else
+          -- info file doesn't exist but entry dir does. delete entry but keep watcher
+          log.debug(string.format("Filesystem event: '%s' removed", info_path_str))
+        end
       end
-    end
-    if do_update then
-      local info_path_str = tostring(info_path)
-      log.debug("Update database for this fs event...")
-      log.debug("Updating: " .. vim.inspect({ path = info_path_str, mtime = mtime }))
-      vim.defer_fn(function()
+      if do_update then
+        local info_path_str = tostring(info_path)
+        log.debug("Update database for this fs event...")
+        log.debug("Updating: " .. vim.inspect({ path = info_path_str, mtime = mtime }))
         data.update_db({ path = info_path_str, mtime = mtime })
-      end, 200)
-    elseif do_unwatch then
-      log.debug("Removing watcher")
-      unwatch_cb()
-    end
+      elseif do_unwatch then
+        log.debug("Removing watcher")
+        unwatch_cb()
+      end
+    end, 200)
   end
 
   -- start the file watcher
@@ -187,7 +200,7 @@ local function start_fs_watch_active_timer()
         log.debug("Taking over file system watching duties")
         data:sync_db()
         start_fs_watchers()
-        log.debug("Start autocmd lo unset db state if this instance stops fs watchers")
+        log.debug("Start autocmd to unset db state if this instance stops fs watchers")
         init_autocmd()
       end
     end)
@@ -209,7 +222,7 @@ function M.start()
   if not does_pid_exist(db.state:get_fw_running()) then
     log.debug("Starting file watchers")
     start_fs_watchers()
-    log.debug("Start autocmd lo unset db state if this instance stops fs watchers")
+    log.debug("Start autocmd to unset db state if this instance stops fs watchers")
     init_autocmd()
   else
     log.debug("This neovim instance will take over file watching if required")

lua/papis/health.lua

@@ -7,7 +7,7 @@
 
 local Path = require("pathlib")
 
-local uv = vim.loop
+local uv = vim.uv
 
 local config = require("papis.config")
 local utils = require("papis.utils")

lua/papis/init.lua

@@ -80,9 +80,13 @@ function M.start()
     return nil
   end
 
+  -- setup commands
+  log.debug("Setting up commands")
+  require("papis.commands").setup()
+
   -- setup enabled modules
   for module_name, _ in pairs(config.enable_modules) do
-    log.trace(module_name .. " is enabled")
+    log.debug(module_name .. " is enabled")
     local has_module, module = pcall(require, "papis." .. module_name)
     if has_module then
       if module.setup then
@@ -91,24 +95,23 @@ function M.start()
     end
   end
 
-  -- setup commands
-  require("papis.commands").setup()
   -- setup keymaps
+  log.debug("Setting up keymaps")
   require("papis.keymaps"):setup()
 
-
   -- check if other neovim instances has file watchers
   local does_pid_exist = require("papis.utils").does_pid_exist
   if not does_pid_exist(db.state:get_fw_running()) then
     -- setup file watchers because no other neovim instance has them
     if config.enable_fs_watcher then
+      log.debug("Setting up file watchers")
       require("papis.fs-watcher"):init()
     end
 
     -- only synchronise the data table if it's not empty
     -- (in that case, we tell users to manually do it because it takes a while)
-    if not db.data:empty() then
-      log.debug("Synchronising the database")
+    if (db.data:empty() == false) and (db.metadata:empty() == false) then
+      log.debug("Setting up/syncronising the database")
       data:sync_db()
     end
   else

lua/papis/papis-storage.lua

@@ -7,7 +7,7 @@
 
 local Path = require("pathlib")
 
-local fs_stat = vim.loop.fs_stat
+local fs_stat = vim.uv.fs_stat
 
 local db = require("papis.sqlite-wrapper")
 if not db then

lua/papis/search/data.lua

@@ -124,8 +124,6 @@ local function init_tbl()
         entry = id,
       },
     })
-
-    require("papis.search").update_precalc(entry)
   end
 end
 

lua/papis/search/init.lua

@@ -24,6 +24,7 @@ setmetatable(papis_entry_display, { __index = entry_display })
 papis_entry_display.truncate = function(a) return a end -- HACK: there must better way to turn this off
 
 local telescope_precalc = {}
+local precalc_last_updated = 0
 
 ---Create a telescope entry for a given db entry
 ---@param entry table #A entry in the library db
@@ -84,19 +85,17 @@ local module_keymaps = {
 
 local M = {}
 
----Updates the precalculated telescope entry for the picker
----@param entry table #The db entry for which to update the telescope entry
-function M.update_precalc(entry)
-  local id = entry.id
-  telescope_precalc[id] = entry_maker(entry)
-end
-
 ---Get precalcuated telescope entries (or create them if they don't yet exist)
 ---@return table #Table with precalculated telescope entries for all db entries
 function M.get_precalc()
-  if vim.tbl_isempty(telescope_precalc) then
+  local db_last_modified = db.state:get_value({ id = 1 }, "db_last_modified")
+  if precalc_last_updated < db_last_modified then
+    log.debug("Updating precalc")
+    precalc_last_updated = db_last_modified
+    telescope_precalc = {}
     local entries = db.data:get()
     for _, entry in ipairs(entries) do
+      -- TODO: only update if mtime for entry indicates a recent change
       local id = entry.id
       telescope_precalc[id] = entry_maker(entry)
     end

lua/papis/sqlite-wrapper.lua

@@ -159,32 +159,70 @@ end
 local function has_schema_changed(new_schema, old_schema)
   local old_schema_okays = sqlite_utils.okeys(old_schema)
   local new_schema_okays = sqlite_utils.okeys(new_schema)
-  -- local len = { new = #new_schema_len, old = #old_schema_len }
   if not vim.deep_equal(old_schema_okays, new_schema_okays) then
     return true
   else
-    return false
+    for _, key in pairs(new_schema_okays) do
+      local normalised_value = {}
+      if new_schema[key] == true then
+        normalised_value = { type = "INTEGER", primary = true, required = true }
+      elseif type(new_schema[key]) == "string" then
+        local new_schema_type = new_schema[key]
+        if new_schema_type ~= "luatable" then
+          new_schema_type = string.upper(new_schema_type)
+        end
+        normalised_value.type = new_schema_type
+      else
+        for k, v in pairs(new_schema[key]) do
+          if k == 1 then
+            local new_schema_type = v
+            if new_schema_type ~= "luatable" then
+              new_schema_type = string.upper(new_schema_type)
+            end
+            normalised_value.type = new_schema_type
+          elseif k == "primary" then
+            normalised_value[k] = v
+          elseif k == "required" then
+            normalised_value[k] = v
+          end
+        end
+        if not vim.tbl_get(new_schema[key], "primary") then
+          normalised_value.primary = false
+        end
+        if not vim.tbl_get(new_schema[key], "required") then
+          normalised_value.required = false
+        end
+      end
+      for k, v in pairs(normalised_value) do
+        if old_schema[key][k] ~= v then
+          return true
+        end
+      end
+    end
   end
+  return false
 end
 
 -- Schemas for all tables
 local schemas = {
   data = config.data_tbl_schema,
   metadata = {
-    id = { "integer", pk = true },
+    id = { "integer", primary = true },
     path = { "text", required = true, unique = true },
     mtime = { "integer", required = true }, -- mtime of the info_yaml
     entry = {
       type = "integer",
       reference = "data.id",
       on_update = "cascade",
       on_delete = "cascade",
+      required = true,
     },
   },
   state = {
     id = true,
     fw_running = { "integer" },
     tag_format = { "text" },
+    db_last_modified = { "integer", default = os.time() },
   },
   config = get_config_tbl_schema(),
 }
@@ -220,8 +258,10 @@ function M:init()
     if self:exists(tbl_name) and (not has_schema_changed(new_schema, old_schema)) then
       self[tbl_name] = self:create_tbl_with_methods(tbl_name)
     else
+      log.debug(string.format("The table schema for '%s' has changed", tbl_name))
       if self:exists(tbl_name) then
         self:drop(tbl_name)
+        self[tbl_name] = self:create_tbl_with_methods(tbl_name)
       end
       self[tbl_name] = self:create_tbl_with_methods(tbl_name)
       if tbl_name == "config" then