feat: including generator meta tag in .readalong files (#226)

* feat: including generator meta tag in .readalong files test: fixing ci conflict resolution: VERSION is now only in readalongs/_version.py TODO: __version__ is no longer defined, and we need a FORMAT_VERSION * fix: resolve the version variable conflicts between format and software * docs: updated DTD version --------- Co-authored-by: Delasie Torkornoo <Delasie.Torkornoo@nrc-cnrc.gc.ca> Co-authored-by: Eric Joanis <eric.joanis@nrc-cnrc.gc.ca>
ReadAlongs · Jul 12, 2024 · 29ec752 · 29ec752
1 parent 4a333bf
commit 29ec752
Show file tree

Hide file tree

Showing 29 changed files with 830 additions and 667 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -121,5 +121,5 @@ jobs:
         # This test will fail if the output encoding is cp1252
         # Warning: the diff line below is PowerShell syntax, not bash!
         run: |
-          echo ćś | readalongs make-xml -l fra - - > cs.readalong
+          echo ćś | readalongs make-xml -l fra - - | findstr /v meta > cs.readalong
           if (diff (cat cs.readalong) (cat test/data/cs-ref.readalong)) { throw "Output did not match reference" }
diff --git a/.gitignore b/.gitignore
@@ -208,3 +208,7 @@ $RECYCLE.BIN/
 *.lnk
 
 # End of https://www.gitignore.io/api/linux,macos,python,windows,visualstudiocode
+
+#temporary file
+.tmp
+.conda
diff --git a/docs/cli-guide.md b/docs/cli-guide.md
@@ -67,7 +67,7 @@ The format of the generated XML is based on [TEI
 Lite](https://tei-c.org/guidelines/customization/lite/) but is
 considerably simplified.  The DTD (document type definition) can be
 found in the ReadAlong Studio source code under
-`readalongs/static/read-along-1.0.dtd`.
+`readalongs/static/read-along-1.1.dtd`.
 
 (dna)=
 

diff --git a/readalongs/_version.py b/readalongs/_version.py
@@ -1 +1,3 @@
 VERSION = "1.1.0"
+
+READALONG_FILE_FORMAT_VERSION = "1.1"
diff --git a/readalongs/align.py b/readalongs/align.py
@@ -18,6 +18,7 @@
 from pympi.Praat import TextGrid
 from webvtt import Caption, WebVTT
 
+from readalongs._version import READALONG_FILE_FORMAT_VERSION, VERSION
 from readalongs.audio_utils import (
     extract_section,
     mute_section,
@@ -175,7 +176,7 @@ def parse_and_make_xml(
     """Parse XML input and run tokenization and G2P.
 
     Args:
-        xml_path (str): Path to input in ReadAlong XML format (see static/read-along-1.0.dtd)
+        xml_path (str): Path to input in ReadAlong XML format (see static/read-along-1.1.dtd)
         config (dict): Optional; ReadAlong-Studio configuration to use
         save_temps (str): Optional; Save temporary files, by default None
         verbose_g2p_warnings (boolean): Optional; display all g2p errors and warnings
@@ -568,7 +569,7 @@ def align_audio(
     """Align an XML input file to an audio file.
 
     Args:
-        xml_path (str): Path to input file in ReadAlong XML format (see static/read-along-1.0.dtd)
+        xml_path (str): Path to input file in ReadAlong XML format (see static/read-along-1.1.dtd)
         audio_path (str): Path to audio input. Must be in a format supported by ffmpeg
         unit (str): Optional; Element to create alignments for, by default 'w'
         bare (boolean): Optional;
@@ -1156,7 +1157,8 @@ def convert_to_xhtml(tokenized_xml, title="Book"):
 
 # TODO: add this <!-- DO NOT USE THIS DATA WITHOUT EXPLICIT PERMISSION --> to template
 RAS_TEMPLATE = """<?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}" />
     <text xml:lang="{{main_lang}}" fallback-langs="{{fallback_langs}}">
         <body>
         {{#pages}}
@@ -1177,7 +1179,7 @@ def convert_to_xhtml(tokenized_xml, title="Book"):
 
 
 def create_ras_from_text(lines: Iterable[str], text_languages=Sequence[str]) -> str:
-    """Create input xml in ReadAlong XML format (see static/read-along-1.0.dtd)
+    """Create input xml in ReadAlong XML format (see static/read-along-1.1.dtd)
         Uses the line sequence to infer paragraph and sentence structure from plain text:
         Assumes a double blank line marks a page break, and a single blank line
         marks a paragraph break.
@@ -1194,6 +1196,8 @@ def create_ras_from_text(lines: Iterable[str], text_languages=Sequence[str]) ->
     kwargs = {
         "main_lang": text_languages[0],
         "fallback_langs": ",".join(text_languages[1:]),
+        "studio_version": VERSION,
+        "format_version": READALONG_FILE_FORMAT_VERSION,
     }
     pages: List[dict] = []
     paragraphs: List[dict] = []
@@ -1223,7 +1227,7 @@ def create_ras_from_text(lines: Iterable[str], text_languages=Sequence[str]) ->
 
 
 def create_input_ras(**kwargs):
-    """Create input xml in ReadAlong XML format (see static/read-along-1.0.dtd)
+    """Create input xml in ReadAlong XML format (see static/read-along-1.1.dtd)
         Uses readlines to infer paragraph and sentence structure from plain text.
         Assumes a double blank line marks a page break, and a single blank line
         marks a paragraph break.

diff --git a/readalongs/static/read-along-1.1.dtd b/readalongs/static/read-along-1.1.dtd
@@ -0,0 +1,103 @@
+<!-- VERSION: 1.1 -->
+<!ELEMENT read-along (meta|text|body|div|span|anchor|silence|graphic|p|s|w)*>
+<!ATTLIST read-along
+ use-assets-folder CDATA #IMPLIED
+ href CDATA #IMPLIED
+ audio CDATA #IMPLIED
+ xml:lang CDATA #IMPLIED
+ language CDATA #IMPLIED
+ lang CDATA #IMPLIED
+ version CDATA #IMPLIED>
+
+<!ELEMENT text (body|div|span|anchor|silence|graphic|p|s|w)*>
+<!ATTLIST text
+ xml:lang CDATA #IMPLIED
+ lang CDATA #IMPLIED
+ fallback-langs CDATA #IMPLIED
+ id CDATA #IMPLIED>
+
+<!ELEMENT body (div|span|anchor|silence|graphic|p|s|w)*>
+<!ATTLIST body
+ xml:lang CDATA #IMPLIED
+ lang CDATA #IMPLIED
+ id CDATA #IMPLIED>
+
+<!ELEMENT anchor EMPTY>
+<!ATTLIST anchor time CDATA #REQUIRED>
+
+<!ELEMENT silence EMPTY>
+<!ATTLIST silence dur CDATA #REQUIRED>
+
+<!ELEMENT graphic EMPTY>
+<!ATTLIST graphic
+ url CDATA #REQUIRED
+ id CDATA #IMPLIED>
+
+<!ELEMENT div (#PCDATA|div|span|anchor|silence|graphic|p|s|w)*>
+<!ATTLIST div
+ xml:lang CDATA #IMPLIED
+ lang CDATA #IMPLIED
+ id CDATA #IMPLIED
+ class CDATA #IMPLIED
+ type CDATA #IMPLIED
+ do-not-align CDATA #IMPLIED
+ time CDATA #IMPLIED
+ dur CDATA #IMPLIED>
+
+<!ELEMENT span (#PCDATA|div|span|anchor|silence|graphic|p|s|w)*>
+<!ATTLIST span
+ xml:lang CDATA #IMPLIED
+ lang CDATA #IMPLIED
+ id CDATA #IMPLIED
+ class CDATA #IMPLIED
+ type CDATA #IMPLIED
+ do-not-align CDATA #IMPLIED
+ time CDATA #IMPLIED
+ dur CDATA #IMPLIED>
+
+<!ELEMENT p (#PCDATA|span|anchor|silence|s|w)*>
+<!ATTLIST p
+ xml:lang CDATA #IMPLIED
+ lang CDATA #IMPLIED
+ id CDATA #IMPLIED
+ class CDATA #IMPLIED
+ do-not-align CDATA #IMPLIED
+ time CDATA #IMPLIED
+ dur CDATA #IMPLIED>
+
+<!ELEMENT s (#PCDATA|span|anchor|silence|w)*>
+<!ATTLIST s
+ xml:lang CDATA #IMPLIED
+ lang CDATA #IMPLIED
+ id CDATA #IMPLIED
+ class CDATA #IMPLIED
+ do-not-align CDATA #IMPLIED
+ time CDATA #IMPLIED
+ dur CDATA #IMPLIED>
+
+<!ELEMENT w (#PCDATA|span|syl)*>
+<!ATTLIST w
+ xml:lang CDATA #IMPLIED
+ effective-g2p-lang CDATA #IMPLIED
+ lang CDATA #IMPLIED
+ id CDATA #IMPLIED
+ class CDATA #IMPLIED
+ do-not-align CDATA #IMPLIED
+ ARPABET CDATA #IMPLIED
+ time CDATA #IMPLIED
+ dur CDATA #IMPLIED>
+
+<!ELEMENT syl (#PCDATA|span)*>
+<!ATTLIST syl
+ xml:lang CDATA #IMPLIED
+ lang CDATA #IMPLIED
+ id CDATA #IMPLIED
+ class CDATA #IMPLIED
+ do-not-align CDATA #IMPLIED
+ ARPABET CDATA #IMPLIED
+ time CDATA #IMPLIED
+ dur CDATA #IMPLIED>
+
+ <!ELEMENT meta EMPTY>
+ <!ATTLIST meta name CDATA #REQUIRED
+   content CDATA #REQUIRED>
diff --git a/readalongs/text/make_package.py b/readalongs/text/make_package.py
@@ -37,7 +37,7 @@
   <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=5.0">
   <meta name="application-name" content="read along">
   <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=5.0">
-  <meta name="generator" content="@readalongs/studio-cli {studio_version}">
+  <meta name="generator" content="@readalongs/studio (cli) {studio_version}">
   <title>{title}</title>
   <script>{js}</script>
   <style attribution="See https://fonts.google.com/attribution for copyrights and font attribution">{fonts}</style>

diff --git a/readalongs/text/util.py b/readalongs/text/util.py
@@ -230,7 +230,7 @@ def copy_file_to_zip(zip_path, origin_path, destination_path):
         <meta charset="UTF-8">
         <meta name="application-name" content="read along">
         <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=5.0">
-        <meta name="generator" content="@readalongs/studio-cli {studio_version}">
+        <meta name="generator" content="@readalongs/studio (cli) {studio_version}">
         <title>{title}</title>
         <!-- Import fonts. Material Icons are needed by the web component -->
         <link href="https://fonts.googleapis.com/css?family=Lato%7CMaterial+Icons%7CMaterial+Icons+Outlined" rel="stylesheet">

diff --git a/readalongs/web_api.py b/readalongs/web_api.py
@@ -41,6 +41,7 @@
 from pydantic import BaseModel, Field
 from starlette.background import BackgroundTask
 
+from readalongs._version import READALONG_FILE_FORMAT_VERSION, VERSION
 from readalongs.align import create_ras_from_text, save_label_files, save_subtitles
 from readalongs.log import LOGGER, capture_logs
 from readalongs.text.add_ids_to_xml import add_ids
@@ -77,7 +78,7 @@
 # Call get_langs() when the server loads to load the languages into memory
 LANGS = get_langs()
 # Get the DTD
-DTDPATH = os.path.join(os.path.dirname(__file__), "static", "read-along-1.0.dtd")
+DTDPATH = os.path.join(os.path.dirname(__file__), "static", "read-along-1.1.dtd")
 with open(DTDPATH) as dtdfh:
     DTD = etree.DTD(dtdfh)
 
@@ -323,7 +324,8 @@ class ConvertRequest(BaseModel):
             dedent(
                 """\
                 <?xml version='1.0' encoding='utf-8'?>
-                <read-along version="1.0">
+                <read-along version="%s">
+    <meta name="generator" content="@readalongs/studio (cli) %s"/>
                     <text xml:lang="dan" fallback-langs="und" id="t0">
                         <body id="t0b0">
                             <div type="page" id="t0b0d0">
@@ -337,6 +339,7 @@ class ConvertRequest(BaseModel):
                         </body>
                     </text>
                 </read-along>"""
+                % (READALONG_FILE_FORMAT_VERSION, VERSION)
             )
         ],
     )

diff --git a/test/data/cs-ref.readalong b/test/data/cs-ref.readalong
@@ -1,5 +1,5 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="1.1">
     <text xml:lang="fra" fallback-langs="und">
         <body>
             <div type="page">

diff --git a/test/data/ej-fra-anchors.readalong b/test/data/ej-fra-anchors.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
     <text xml:lang="fra">
         <body>
             <div type="page">

diff --git a/test/data/ej-fra-anchors2.readalong b/test/data/ej-fra-anchors2.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
     <text xml:lang="fra">
         <anchor time=".5s"/>
         <body>

diff --git a/test/data/ej-fra-dna.readalong b/test/data/ej-fra-dna.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
     <text xml:lang="fra">
         <body>
             <div type="page">

diff --git a/test/data/ej-fra-package.readalong b/test/data/ej-fra-package.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
     <text xml:lang="fra">
         <body>
             <div type="page">

diff --git a/test/data/ej-fra-silence-bad.readalong b/test/data/ej-fra-silence-bad.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
     <text xml:lang="fra">
         <body>
             <div type="page">

diff --git a/test/data/ej-fra-silence.readalong b/test/data/ej-fra-silence.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
     <text xml:lang="fra">
         <body>
             <div type="page">

diff --git a/test/data/ej-fra-subword.readalong b/test/data/ej-fra-subword.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
     <!-- To exclude any element from alignment, add the do-not-align="true" attribute to
          it, e.g., <p do-not-align="true">...</p>, or
          <s>Some text <foo do-not-align="true">do not align this</foo> more text</s> -->

diff --git a/test/data/ej-fra-translated.readalong b/test/data/ej-fra-translated.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
     <!-- To exclude any element from alignment, add the do-not-align="true" attribute to
          it, e.g., <p do-not-align="true">...</p>, or
          <s>Some text <foo do-not-align="true">do not align this</foo> more text</s> -->

diff --git a/test/data/ej-fra.readalong b/test/data/ej-fra.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
     <text xml:lang="fra">
         <body>
             <div type="page">

diff --git a/test/data/fra-prepared.readalong b/test/data/fra-prepared.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}" />
     <text xml:lang="fra" fallback-langs="und">
         <body>
             <div type="page">

diff --git a/test/data/fra-tokenized.readalong b/test/data/fra-tokenized.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
     <text xml:lang="fra">
         <body>
             <div type="page">

diff --git a/test/data/mixed-langs.readalong b/test/data/mixed-langs.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
     <text>
         <body>
             <div type="page">

diff --git a/test/data/patrickxtlan.readalong b/test/data/patrickxtlan.readalong
@@ -1,5 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<read-along version="1.0">
+<read-along version="{{format_version}}">
+    <meta name="generator" content="@readalongs/studio (cli) {{studio_version}}"/>
   <text>
     <body>
     <p>