Merge pull request #4989 from greenriver/release-144

Release 144

.github/workflows/build_images.yml

@@ -45,7 +45,7 @@ jobs:
             tags: |
               type=sha,prefix=githash-
               type=ref,event=branch,prefix=branch-
-              # type=raw,event=branch,value=branch-{{branch}}-{{sha}}
+              type=raw,event=branch,value=branch-{{branch}}-{{sha}}
 
     steps:
       - name: Checkout

app/jobs/confidence/add_enrollment_change_history_job.rb

@@ -4,6 +4,8 @@
 # License detail: https://github.com/greenriver/hmis-warehouse/blob/production/LICENSE.md
 ###
 
+# Snapshot client enrollment histories for auditing
+# * note this job seems to stop and re-queue itself frequently, probably to free worker processes
 module Confidence
   class AddEnrollmentChangeHistoryJob < BaseJob
     queue_as ENV.fetch('DJ_LONG_QUEUE_NAME', :long_running)

app/mailers/account_mailer.rb

@@ -10,7 +10,7 @@ class AccountMailer < Devise::Mailer
   ActionMailer::Base.register_interceptor CloudwatchEmailInterceptor if ENV['SES_MONITOR_OUTGOING_EMAIL'] == 'true'
 
   def invitation_instructions(record, action, opts = {})
-    opts[:subject] = Translation.translate('Boston DND Warehouse') + ': Account Activation Instructions'
+    opts[:subject] = Translation.translate('Open Path HMIS Warehouse') + ': Account Activation Instructions'
     super
   end
 end

app/models/concerns/client_file_base.rb

@@ -16,7 +16,9 @@ module ClientFileBase
 
     has_one_attached :client_file
 
-    validates_presence_of :name
+    # The file_exists_and_not_too_large validation checks that the client_file exists, so we can rely on that and only validate presence of name if the file has a client_file attachment.
+    # This avoids an awkward double-error "Name can't be blank" if the file is not uploaded or has expired.
+    validates_presence_of :name, if: -> { client_file.attached? }
     validate :file_exists_and_not_too_large
     validate :note_if_other
 
@@ -49,7 +51,7 @@ def tags
     end
 
     def file_exists_and_not_too_large
-      errors.add :client_file, full_message: 'No uploaded file found' if (client_file.byte_size || 0) < 100
+      errors.add :client_file, full_message: 'No uploaded file found.' if (client_file.byte_size || 0) < 100
       errors.add :client_file, full_message: 'File size should be less than 4 MB' if (client_file.byte_size || 0) > 4.megabytes
     end
 

app/models/concerns/user_concern.rb

@@ -300,7 +300,7 @@ def credential_options
     end
 
     def two_factor_label
-      label = Translation.translate('Boston DND HMIS Warehouse')
+      label = Translation.translate('Open Path HMIS Warehouse')
       Rails.env.production? ? label : "#{label} [#{Rails.env}]"
     end
 

app/models/grda_warehouse/data_source.rb

@@ -497,36 +497,41 @@ def unprocessed_enrollment_count
     @unprocessed_enrollment_count ||= enrollments.unprocessed.joins(:project, :destination_client).count
   end
 
-  # Have we received the expected number of files at least once over the past 48 hours?
-  # there is an assumption that within
-  # return the date of the most-recent fully successful import
-  def stalled_since?(date)
-    return nil unless date.present?
+  # Returns the date of the most recent fully successful import if the import is stalled, nil if it is not stalled
+  # @return [Date, nil] The date the import stalled, or nil if not stalled.
+  def stalled_date
     return nil if import_paused
     return nil unless hmis_import_config&.active
 
     # hmis_import_config.file_count is the expected number of uploads for a given day
     # fetch the expected number, and confirm they all arrived within a 24 hour window
     most_recent_uploads = uploads.completed.
-      # limit look back to 6 months to improve performance
-      where(user_id: User.system_user.id, completed_at: 6.months.ago.to_date..Date.current).
-      order(created_at: :desc).
-      select(:id, :data_source_id, :user_id, :completed_at).
+      # limit look back to 6 months to improve performance, but to potentially highlight missing data
+      where(user_id: User.system_user.id, completed_at: 6.months.ago..Time.current).
+      order(completed_at: :desc, created_at: :desc).
+      select(:id, :data_source_id, :user_id, :completed_at, :created_at).
+      distinct.
       first(hmis_import_config.file_count)
-    return nil unless most_recent_uploads
+    # We didn't find any uploads in the last 6 months, assume this isn't connected yet
+    return nil unless most_recent_uploads.present?
+
+    min_completion_time = most_recent_uploads.minimum(:completed_at)
+    received_files_count = most_recent_uploads.count
 
-    most_recent_upload = most_recent_uploads.first
-    previously_completed_upload = most_recent_uploads.last
-    # Check that the expected number of files arrived within a 24 hour window, otherwise we might be looking
-    # at two partial runs
-    # If we only expect one file, then first and last will be the same and time_diff will be 0.0
-    return nil if most_recent_upload.blank? || previously_completed_upload.blank?
+    # If we only expected one file
+    if hmis_import_config.file_count == 1
+      # and it came in the last 24 hours, we're good
+      return nil if min_completion_time > 24.hours.ago
+
+      # if not, return the last time we received a file
+      return min_completion_time.to_date
+    end
 
-    time_diff = most_recent_upload.completed_at - previously_completed_upload.completed_at
-    return most_recent_upload.completed_at unless time_diff < 24.hours.to_i
-    return nil if most_recent_upload.completed_at > 48.hours.ago
+    # If we processed the expected number of files within a 24 hour period, we're good
+    return nil if min_completion_time > 24.hours.ago && received_files_count >= hmis_import_config.file_count
 
-    most_recent_upload.completed_at.to_date
+    # Note the last time we received a file
+    min_completion_time.to_date
   end
 
   def self.stalled_imports?(user)
@@ -537,7 +542,7 @@ def self.stalled_imports?(user)
 
         most_recently_completed = data_source.import_logs.maximum(:completed_at)
         if most_recently_completed.present?
-          stalled = true if data_source.stalled_since?(most_recently_completed)
+          stalled = true if data_source.stalled_date.present?
         end
       end
 

app/models/grda_warehouse/enrollment_change_history.rb

@@ -4,6 +4,19 @@
 # License detail: https://github.com/greenriver/hmis-warehouse/blob/production/LICENSE.md
 ###
 
+# GrdaWarehouse::EnrollmentChangeHistory
+#
+# Audit system that tracks changes to enrollment data over time to support data quality monitoring and troubleshooting.
+#
+# Answers questions like:
+# - "client had an enrollment here last month, but now it's gone?"
+# - "enrollment had 300 days of service and now only has 10, why?"
+#
+# Notes
+# - Access to these records is restricted by explicit permission
+# - Records are generated asynchronously in batches
+# - Records retained for 6 months (configurable via RETENTION_DURATION)
+
 module GrdaWarehouse
   class EnrollmentChangeHistory < GrdaWarehouseBase
     belongs_to :client, class_name: 'GrdaWarehouse::Hud::Client'
@@ -32,8 +45,9 @@ def self.generate_for_date!(date: Date.current)
         perform_later(date: date.to_s)
     end
 
+    # client_ids: destination client ids
     def self.create_for_clients_on_date! client_ids:, date:
-      clients = GrdaWarehouse::Hud::Client.destination.where(id: client_ids)
+      clients = GrdaWarehouse::Hud::Client.destination.where(id: client_ids).preload(:processed_service_history)
       rows = clients.map do |client|
         attributes_for_client_on_date(client: client, date: date) rescue nil # rubocop:disable Style/RescueModifier
       end.compact
@@ -42,17 +56,21 @@ def self.create_for_clients_on_date! client_ids:, date:
     end
 
     def self.attributes_for_client_on_date client:, date:
+      now = Time.current
       attributes_for_client = {
         client_id: client.id,
         on: date,
         # are timestamps and version columns used? This table is large; perhaps they could be dropped for efficiency
-        created_at: Time.now,
-        updated_at: Time.now,
+        created_at: now,
+        updated_at: now,
         version: 1,
       }
-      attributes_for_client[:residential] = client.enrollments_for_rollup(en_scope: client.scope_for_residential_enrollments(current_user), user: User.setup_system_user).to_json rescue '[]' # rubocop:disable Style/RescueModifier
-      attributes_for_client[:other] = client.enrollments_for_rollup(en_scope: client.scope_for_other_enrollments(current_user), user: User.setup_system_user).to_json rescue '[]' # rubocop:disable Style/RescueModifier
-      attributes_for_client[:days_homeless] = client.days_homeless rescue 0 # rubocop:disable Style/RescueModifier
+
+      # System User is correct here
+      user = User.setup_system_user
+      attributes_for_client[:residential] = client.enrollments_for_rollup(en_scope: client.scope_for_residential_enrollments(user), user: user).to_json
+      attributes_for_client[:other] = client.enrollments_for_rollup(en_scope: client.scope_for_other_enrollments(user), user: user).to_json
+      attributes_for_client[:days_homeless] = client.days_homeless
       return attributes_for_client
     end
   end

app/models/grda_warehouse/hud/inventory.rb

@@ -88,24 +88,6 @@ class Inventory < Base
       where(HouseholdType: HOUSEHOLD_TYPES[:child_only])
     end
 
-    scope :overridden, -> do
-      scope = where(Arel.sql('1=0'))
-      override_columns.each_key do |col|
-        scope = scope.or(where.not(col => nil))
-      end
-      scope
-    end
-
-    TodoOrDie('Remove override_columns method and columns from the database', by: '2025-01-23')
-    # If any of these are not blank, we'll consider it overridden
-    def self.override_columns
-      {
-        coc_code_override: :CoCCode,
-        inventory_start_date_override: :InventoryStartDate,
-        inventory_end_date_override: :InventoryEndDate,
-      }
-    end
-
     def for_export
       row = HmisCsvTwentyTwentyTwo::Exporter::Inventory::Overrides.apply_overrides(self)
       row = HmisCsvTwentyTwentyTwo::Exporter::Inventory.adjust_keys(row)

app/models/grda_warehouse/hud/project.rb

@@ -145,12 +145,6 @@ def night_by_night?
       es_nbn?
     end
 
-    # DEPRECATED_FY2024 - remove this once the transition 2024 is complete
-    # Make some tests work
-    def es_nbn_pre_2024?
-      tracking_method_to_use == 3 && project_type_to_use == 1
-    end
-
     scope :confidential, -> do
       joins(:organization).where(p_t[:confidential].eq(true).or(o_t[:confidential].eq(true)))
     end
@@ -194,9 +188,7 @@ def es_nbn_pre_2024?
     end
 
     def coc_funded?
-      return self.ContinuumProject == 1 if hud_continuum_funded.nil?
-
-      hud_continuum_funded
+      self.ContinuumProject == 1
     end
 
     # NOTE: Careful, this returns duplicates as it joins inventories.
@@ -345,29 +337,6 @@ def serves_children?
       end
     end
 
-    scope :overridden, -> do
-      scope = where(Arel.sql('1=0'))
-      override_columns.each_key do |col|
-        scope = scope.or(where.not(col => nil))
-      end
-      scope
-    end
-
-    # TODO: This should be removed when all overrides have been removed
-    TodoOrDie('Remove override_columns method and columns from the database', by: '2025-01-23')
-    # If any of these are not blank, we'll consider it overridden
-    def self.override_columns
-      {
-        act_as_project_type: :ProjectType,
-        hud_continuum_funded: :ContinuumProject,
-        housing_type_override: :HousingType,
-        operating_start_date_override: :OperatingStartDate,
-        operating_end_date_override: :OperatingEndDate,
-        hmis_participating_project_override: :HMISParticipatingProject,
-        target_population_override: :TargetPopulation,
-      }
-    end
-
     def self.can_see_all_projects?(user)
       visible_count = viewable_by(user).distinct.count
       visible_count.positive? && visible_count == all.count
@@ -877,11 +846,6 @@ def pay_for_success?
       funders.map(&:pay_for_success?).any?
     end
 
-    # DEPRECATED_FY2024 no longer used in FY2024
-    def tracking_method_to_use
-      tracking_method_override.presence || self.TrackingMethod
-    end
-
     def human_readable_project_type
       HudUtility2024.project_type(project_type_to_use)
     end

app/models/grda_warehouse/hud/project_coc.rb

@@ -95,25 +95,6 @@ class ProjectCoc < Base
       # END_ACL
     end
 
-    scope :overridden, -> do
-      scope = where(Arel.sql('1=0'))
-      override_columns.each_key do |col|
-        scope = scope.or(where.not(col => nil))
-      end
-      scope
-    end
-
-    TodoOrDie('Remove override_columns method and columns from the database', by: '2025-01-23')
-    # If any of these are not blank, we'll consider it overridden
-    def self.override_columns
-      {
-        hud_coc_code: :CoCCode,
-        geography_type_override: :GeographyType,
-        geocode_override: :Geocode,
-        zip_override: :Zip,
-      }
-    end
-
     def self.zip_code_shapes
       joins(<<~SQL)
         INNER JOIN shape_zip_codes ON ( shape_zip_codes.zcta5ce10 = "ProjectCoC"."Zip" OR shape_zip_codes.zcta5ce10 = "ProjectCoC"."zip_override")

app/models/hud_reports/report_instance.rb

@@ -35,8 +35,8 @@ def self.from_filter(filter, report_name, build_for_questions:)
         remaining_questions: build_for_questions,
         user_id: filter.user_id,
         project_ids: filter.effective_project_ids,
-        start_date: filter.start.to_date,
-        end_date: filter.end.to_date,
+        start_date: filter.start&.to_date,
+        end_date: filter.end&.to_date,
         coc_codes: filter.coc_codes,
         options: filter.to_h,
       )

app/models/report_generators/pit/fy2018/base.rb

@@ -845,7 +845,7 @@ def chronic_scope
 
     def sh_cols
       @sh_cols ||= {
-        project_type: act_as_project_overlay,
+        project_type: shs_t[:project_type].as('project_type'),
         client_id: she_t[:client_id].as('client_id'),
         enrollment_group_id: she_t[:enrollment_group_id].as('enrollment_group_id'),
         age: shs_t[:age].as('age'),
@@ -860,10 +860,6 @@ def sh_cols
       }
     end
 
-    def act_as_project_overlay
-      nf('COALESCE', [p_t[:act_as_project_type], shs_t[:project_type]]).as('project_type')
-    end
-
     def client_columns
       {
         PersonalID: c_t[:PersonalID].as('PersonalID'),

app/models/translation.rb

@@ -105,9 +105,6 @@ def self.known_translations
       'BH CP Request for Care Plan Signature',
       'Backup Plan Emergency Note',
       'Bed Utilization Percentages',
-      'Boston DND HMIS Warehouse',
-      'Boston DND Warehouse',
-      'Boston HMIS staff at DND',
       'Boston Reports Configuration',
       'Built For Zero Monthly Report',
       'Bulk set Health Prioritization for CAS.',
@@ -1245,7 +1242,7 @@ def self.known_translations
       'Child-Only Households',
       'Children under age 18 in household',
       'Chronically Homeless for CAS',
-      'City of Boston DND Warehouse',
+      'Open Path HMIS Warehouse',
       'Clean/sober for at least one year',
       'Client Activity Report',
       'Client Contact Locations',
@@ -1314,7 +1311,7 @@ def self.known_translations
       'Document Ready',
       'Earning a living wage ($13 or more)',
       'Employed for 3 or more months',
-      'Ending Veteran & Chronic Homelessness in Boston',
+      'Ending Homelessness',
       'Enrolled in PH',
       'Enrolled in sheltered homeless project (ES, TH, SH)',
       'Enrolled in unsheltered homeless project (SO)',
@@ -1422,7 +1419,6 @@ def self.known_translations
       'Location Type',
       'MA YYA Follow Up Report',
       'MA YYA Report',
-      'MA-500 Boston Continuum of Care FY2022 Renewal Project Scoring Tool',
       'Manually entered date at which the client became document ready',
       'Median Length of Time from CE Project Entry to Housing Referral',
       'Median Length of Time from Housing Referral to Housing Start',
@@ -1593,7 +1589,6 @@ def self.known_translations
       'TX-601 established Community-wide performance expectations in 2016 to allocate assistance as effectively as possible, prioritizing services to those that need it the most.',
       'TX-601 established Community-wide performance expectations in 2016 to allocate assistance as effectively as possible, prioritizing services to those that need it the most. Projects are allowed to reject up to 10% of  CES referrals for reasons stated in the CES Operation Manual.',
       'Text Message Queue',
-      'The Boston DND Warehouse is operated by the Department of Neighborhood Development as the lead agency of the Boston Continuum of Care.',
       'The HMIS Warehouse can be used for homeless client care coordination and reporting.',
       'The measure indicates if agency is participating in CoC-related activities',
       'This is a Family VI-SPDAT',
@@ -1797,7 +1792,7 @@ def self.known_translations
       'pre-placement',
       'search',
       'stabilization',
-      'the Boston Emergency Shelter Commission',
+      'the Emergency Shelter Commission',
       'the date of the activity is missing',
       'too many months with non-outreach activities and no signed careplan',
       'too many months with outreach activities',

app/views/application/_inactive_session_modal.haml

@@ -11,7 +11,7 @@
   .session_expiry__content.d-none{data: {'inactive-session-modal-target': 'alert'}}
     .session_expiry__alert-box
       .p-2
-        - title = Translation.translate('Boston DND Warehouse')
+        - title = Translation.translate('Open Path HMIS Warehouse')
         - if title.present?
           %h4.mt-0= title
         .mb-4.alert.alert-warning.w-100{data: {'inactive-session-modal-target': 'alertMessage'}}