Created data transformers for #3888

Gemfile

@@ -63,7 +63,7 @@ gem 'charlock_holmes'
 
 gem 'edtf'
 gem 'edtf-humanize'
-
+gem 'diffy'
 gem 'terser'
 group :assets do
   gem 'uglifier'

Gemfile.lock

@@ -202,6 +202,7 @@ GEM
       devise (>= 4.7.0)
       railties (>= 5.2.0)
     diff-lcs (1.5.0)
+    diffy (3.4.2)
     docile (1.4.0)
     easy_translate (0.5.1)
       thread
@@ -632,6 +633,7 @@ DEPENDENCIES
   devise
   devise-encryptable
   devise_masquerade (~> 1.2.0)
+  diffy
   easy_translate
   edtf
   edtf-humanize

app/assets/javascripts/application.js

@@ -319,6 +319,7 @@ function addOptions(selector, enabled_index){
   var parentTr = selector.parentElement.parentElement;
   var optionsObj = $(parentTr).find('td .field-options')[0];
   var index = selector.options.selectedIndex;
+  logger.debug("addOptions");
   if (index == enabled_index){
     $(optionsObj).prop('disabled', false);
   } else {

app/assets/stylesheets/codemirror/lib/codemirror.css

@@ -3,17 +3,9 @@
 .CodeMirror {
   /* Set height, width, borders, and global font properties here */
   font-family: monospace;
-  height: auto;
+  height: 300px;
   color: black;
-}
-/* Set the direction of the lenguage for ltr and rtl */
-.rtl {
-  text-align: right;
-  padding: 20px;
-}
-
-.ltr {
-  text-align: left;
+  direction: ltr;
 }
 
 /* PADDING */
@@ -68,20 +60,13 @@
 .cm-fat-cursor div.CodeMirror-cursors {
   z-index: 1;
 }
-.cm-fat-cursor-mark {
-  background-color: rgba(20, 255, 20, 0.5);
-  -webkit-animation: blink 1.06s steps(1) infinite;
-  -moz-animation: blink 1.06s steps(1) infinite;
-  animation: blink 1.06s steps(1) infinite;
-}
-.cm-animate-fat-cursor {
-  width: auto;
-  border: 0;
-  -webkit-animation: blink 1.06s steps(1) infinite;
-  -moz-animation: blink 1.06s steps(1) infinite;
-  animation: blink 1.06s steps(1) infinite;
-  background-color: #7e7;
-}
+.cm-fat-cursor .CodeMirror-line::selection,
+.cm-fat-cursor .CodeMirror-line > span::selection, 
+.cm-fat-cursor .CodeMirror-line > span > span::selection { background: transparent; }
+.cm-fat-cursor .CodeMirror-line::-moz-selection,
+.cm-fat-cursor .CodeMirror-line > span::-moz-selection,
+.cm-fat-cursor .CodeMirror-line > span > span::-moz-selection { background: transparent; }
+.cm-fat-cursor { caret-color: transparent; }
 @-moz-keyframes blink {
   0% {}
   50% { background-color: transparent; }
@@ -179,6 +164,7 @@ div.CodeMirror span.CodeMirror-nonmatchingbracket {color: #a22;}
   height: 100%;
   outline: none; /* Prevent dragging from highlighting the element */
   position: relative;
+  z-index: 0;
 }
 .CodeMirror-sizer {
   position: relative;

app/models/page.rb

@@ -541,19 +541,26 @@ def contributors
     users
   end
 
-  def has_alto?
-    File.exists?(alto_path)
+  def has_ai_plaintext?
+    File.exists?(ai_plaintext_path)
   end
 
-  def create_alto
-    if File.exists? original_htr_path
-      # convert the htr to alto
-      # write the alto to the alto path
+  def ai_plaintext
+    if has_ai_plaintext?
+      File.read(ai_plaintext_path)
     else
-      # call a service to create HTR
-      PageProcessor.new(self).submit_process
+      ""
     end
+  end
 
+  def ai_plaintext=(text)
+    FileUtils.mkdir_p(File.dirname(ai_plaintext_path)) unless Dir.exist? File.dirname(ai_plaintext_path)
+    File.write(ai_plaintext_path, text)
+  end
+
+
+  def has_alto?
+    File.exists?(alto_path)
   end
 
 
@@ -582,6 +589,9 @@ def image_url_for_download
   end
 
   private
+  def ai_plaintext_path
+    File.join(Rails.root, 'public', 'text', self.work_id.to_s, "#{self.id}_ai_plaintext.txt")
+  end
 
   def alto_path
     File.join(Rails.root, 'public', 'text', self.work_id.to_s, "#{self.id}_alto.xml")

app/views/shared/_codemirror.html.erb

@@ -6,6 +6,26 @@
 <%=javascript_include_tag "codemirror/addon/hint/xml-hint"%>
 <%=javascript_include_tag "codemirror-buttons/buttons"%>
 
+<% if @page && !@page.ai_plaintext.blank? %>
+  <a href="#" id="copy-ai-plaintext" class="btn btn-default btn-sm" data-toggle="tooltip" data-placement="top"><%= t('.copy_ai_plaintext') %></a>
+  <script>
+    var ai_plaintext = <%= raw(@page.ai_plaintext.to_json) %>;
+    var ai_plaintext_caveat = <%= raw(t('.ai_plaintext_caveat').to_json) %>;
+
+    function appendAiPlaintext(cm, pred) {
+      var currentText = cm.getValue();
+      // append ai_plaintext to the end of the current text
+      cm.setValue(currentText + ai_plaintext_caveat + "\n" + ai_plaintext);
+    }; 
+    document.getElementById("copy-ai-plaintext").addEventListener("click", function() {
+      appendAiPlaintext(myCodeMirror);
+      // now disable the button
+      $('#copy-ai-plaintext').prop('disabled', true);
+    });
+
+  </script>
+<% end %>
+
 <script>
   var button_config = [
     <% @collection.editor_buttons.each do |button| %>

app/views/shared/_osd_div.html.slim

@@ -196,10 +196,10 @@
         const btnLocked = document.getElementById('toggle-locked');
         btnLocked.addEventListener('click', () => {
           if (textlayer.isLocked()) {
-            btnLocked.innerHTML = '#{svg_symbol "#icon-clipboard", class: "icon"}}';
+            btnLocked.innerHTML = '#{svg_symbol "#icon-clipboard", class: "icon"}';
             textlayer.setLocked(false);
           } else {
-            btnLocked.innerHTML = '#{svg_symbol "#icon-cursor-text", class: "icon"}}';
+            btnLocked.innerHTML = '#{svg_symbol "#icon-cursor-text", class: "icon"}';
             textlayer.setLocked(true);
           }
         });

app/views/transcribe/_save_buttons.html.slim

@@ -1,5 +1,7 @@
 .flex-toolbar_group
   -unless @preview_xml
+    -if @page && !@page.ai_plaintext.blank? && @collection.text_entry?
+      =button_tag t('.copy_ai_plaintext'), type: 'button', id: 'copy-ai-plaintext', title: "#{t('.copy_ai_plaintext_tooltip')}"
     =button_tag t('.preview'), :name => 'preview', type: 'submit', title: "#{t('.preview_tooltip')}"
   -else
     =button_tag t('.edit'), :name => 'edit', type: 'submit', title: "#{t('.edit_tooltip')}"

config/locales/shared/shared-de.yml

@@ -11,6 +11,8 @@ de:
     codemirror:
       abbr_description: Kennzeichnet Abkürzungen in einem Text mit Erweiterungen im expan-Attribut.
       add_description: Markiert Text, der nachträglich vom Autor/von der Autorin eingefügt wurde.
+      ai_plaintext_caveat: "<!-- Der folgende Text wurde durch künstliche Intelligenz generiert und enthält wahrscheinlich Fehler. Einige offensichtliche Fehler wurden durch 🤔-Symbole ersetzt, kniffligere Fehler werden jedoch möglicherweise nicht markiert. (Bitte löschen Sie diese Warnung.) -->"
+      copy_ai_plaintext: KI-Text
       date_description: Markiert Datumsangaben mit standardisierter Form im when-Attribut.
       del_description: Markiert Text, der durch Löschen oder Durchstreichen entfernt wurde.
       expan_description: Kennzeichnet erweiterte Abkürzungen mit Originalformen im abbr-Attribut.

config/locales/shared/shared-en.yml

@@ -11,6 +11,8 @@ en:
     codemirror:
       abbr_description: Marks abbreviations in a text with expansions in the expan attribute.
       add_description: Marks text inserted after-the-fact by the writer.
+      ai_plaintext_caveat: "<!-- The following text was generated by artificial intelligence and probably contains errors.  Some obvious errors have been replaced with 🤔 symbols, but trickier ones may not be marked. (Please delete this warning.) -->"
+      copy_ai_plaintext: AI Text
       date_description: Marks dates, with standardized form in the when attribute.
       del_description: Marks text that has been removed by erasure or strike-through.
       expan_description: Marks expanded abbreviations with original forms in the abbr attribute.

config/locales/shared/shared-es.yml

@@ -11,6 +11,8 @@ es:
     codemirror:
       abbr_description: Marca las abreviaturas abreviaturas en un texto con expansiones en el atributo expan.
       add_description: Marca el texto insertado después por el escritor.
+      ai_plaintext_caveat: "<!-- El siguiente texto fue generado por inteligencia artificial y probablemente contenga errores. Algunos errores obvios se han reemplazado con símbolos 🤔, pero es posible que los más complicados no estén marcados. (Elimine esta advertencia). -->"
+      copy_ai_plaintext: Texto de IA
       date_description: Marca la fecha, con forma estandarizada según el atributo cuándo.
       del_description: Marca el texto que ha sido borrado o tachado.
       expan_description: Marcas las abreviaturas ampliadas con formularios originales en el atributo abbr.

config/locales/shared/shared-fr.yml

@@ -11,6 +11,8 @@ fr:
     codemirror:
       abbr_description: Marque les abréviations dans un texte avec des extensions dans l'attribut expan.
       add_description: Marque le texte inséré après coup par l'auteur.
+      ai_plaintext_caveat: "<!-- Le texte suivant a été généré par l'intelligence artificielle et contient probablement des erreurs. Certaines erreurs évidentes ont été remplacées par des symboles 🤔, mais les plus délicates ne peuvent pas être marquées. (Veuillez supprimer cet avertissement.) -->"
+      copy_ai_plaintext: Texte IA
       date_description: Marque les dates, avec une forme standardisée dans l'attribut quand.
       del_description: Marque le texte qui a été supprimé par effacement ou barré.
       expan_description: Marque les abréviations développées avec les formes originales dans l'attribut abbr.

config/locales/shared/shared-pt.yml

@@ -11,6 +11,8 @@ pt:
     codemirror:
       abbr_description: Marca as abreviações em um texto com expansões no atributo expan.
       add_description: Marca o texto inserido após o fato pelo escritor.
+      ai_plaintext_caveat: "<!-- O texto a seguir foi gerado por inteligência artificial e provavelmente contém erros. Alguns erros óbvios foram substituídos por símbolos 🤔, mas os mais complicados podem não ser marcados. (Por favor, exclua este aviso.) -->"
+      copy_ai_plaintext: Texto de IA
       date_description: Marca datas, com forma padronizada no atributo quando.
       del_description: Marca o texto que foi removido por apagamento ou tachado.
       expan_description: Marca abreviações expandidas com formas originais no atributo abbr.

config/locales/transcribe/transcribe-de.yml

@@ -46,6 +46,8 @@ de:
       approve_to_transcribed_tooltip: Diese Seite speichern und freigeben
       autolink: Autolink
       autolink_tooltip: Entitäten-Tags automatisch vorschlagen
+      copy_ai_plaintext: KI-Text
+      copy_ai_plaintext_tooltip: Kopieren Sie KI-generierten Text in den Editor
       done: Fertig
       edit: Bearbeiten
       edit_tooltip: Bearbeite diese Seite

config/locales/transcribe/transcribe-en.yml

@@ -46,6 +46,8 @@ en:
       approve_to_transcribed_tooltip: Save and approve this page
       autolink: Autolink
       autolink_tooltip: Suggest subject tags automatically
+      copy_ai_plaintext: AI Text
+      copy_ai_plaintext_tooltip: Copy AI-generated text to the editor
       done: Done
       edit: Edit
       edit_tooltip: Edit this page

config/locales/transcribe/transcribe-es.yml

@@ -46,6 +46,8 @@ es:
       approve_to_transcribed_tooltip: Guardar y aprobar esta página
       autolink: Enlace automático
       autolink_tooltip: Sugerir etiquetas de asunto automáticamente
+      copy_ai_plaintext: Texto de IA
+      copy_ai_plaintext_tooltip: Copie el texto generado por IA al editor
       done: Terminado
       edit: Editar
       edit_tooltip: Edita esta página

config/locales/transcribe/transcribe-fr.yml

@@ -46,6 +46,8 @@ fr:
       approve_to_transcribed_tooltip: Enregistrer et approuver cette page
       autolink: Lien automatique
       autolink_tooltip: Suggérer automatiquement des balises de sujet
+      copy_ai_plaintext: Texte IA
+      copy_ai_plaintext_tooltip: Copiez le texte généré par l'IA dans l'éditeur
       done: Terminé
       edit: Éditer
       edit_tooltip: Modifier cette page

config/locales/transcribe/transcribe-pt.yml

@@ -46,6 +46,8 @@ pt:
       approve_to_transcribed_tooltip: Salve e aprove esta página
       autolink: Autolink
       autolink_tooltip: Sugerir tags de assunto automaticamente
+      copy_ai_plaintext: Texto de IA
+      copy_ai_plaintext_tooltip: Copie o texto gerado pela IA para o editor
       done: Feito
       edit: Editar
       edit_tooltip: Edite essa página

db/migrate/20231230192412_migrate_unicode_encoding.rb

@@ -0,0 +1,16 @@
+class MigrateUnicodeEncoding < ActiveRecord::Migration[6.0]
+  def change
+    p Time.now
+    execute "ALTER TABLE pages CHANGE source_text source_text mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+    execute "ALTER TABLE pages CHANGE xml_text xml_text mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+    execute "ALTER TABLE pages CHANGE source_translation source_translation mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+    execute "ALTER TABLE pages CHANGE xml_translation xml_translation mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+    execute "ALTER TABLE pages CHANGE search_text search_text text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+    p Time.now
+    execute "ALTER TABLE page_versions CHANGE transcription transcription mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+    execute "ALTER TABLE page_versions CHANGE xml_transcription xml_transcription mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+    execute "ALTER TABLE page_versions CHANGE source_translation source_translation text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+    execute "ALTER TABLE page_versions CHANGE xml_translation xml_translation text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+
+  end
+end

db/migrate/20231231150510_migrate_unicode_encoding_notes_tables.rb

@@ -0,0 +1,5 @@
+class MigrateUnicodeEncodingNotesTables < ActiveRecord::Migration[6.0]
+  def change
+    execute "ALTER TABLE notes CHANGE body body mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+  end
+end