issue #124: adding KBR version and KBR related subfields.

pkiraly · Jan 26, 2022 · e01cd49 · e01cd49
1 parent 40e8c5f
commit e01cd49
Show file tree

Hide file tree

Showing 230 changed files with 2,066 additions and 9 deletions.
diff --git a/catalogues/kbr.sh b/catalogues/kbr.sh
@@ -2,9 +2,9 @@
 
 . ./setdir.sh
 NAME=kbr
-MARC_DIR=${BASE_INPUT_DIR}/kbr
-TYPE_PARAMS="--emptyLargeCollectors"
-MASK=KBR_20210713_B_0?.mrc.gz
+MARC_DIR=${BASE_INPUT_DIR}/kbr/current
+TYPE_PARAMS="--emptyLargeCollectors --marcVersion KBR --marcxml --ignorableFields 590,591,592,593,594,595,596,659,900,911,912,916,940,941,942,944,945,946,948,949,950,951,952,953,954,970,971,972,973,975,977,988,989"
+MASK=kbr-*.gz
 
 . ./common-script
 

diff --git a/common-script b/common-script
@@ -277,6 +277,7 @@ do_export_schema_files() {
   ./export-schema --withSubfieldCodelists > marc-schema/marc-schema.json
   ./export-schema --withSubfieldCodelists --solrFieldType human-readable --withSelfDescriptiveCode > marc-schema/marc-schema-with-solr.json
   ./export-schema --withSubfieldCodelists --solrFieldType human-readable --withSelfDescriptiveCode --withLocallyDefinedFields > marc-schema/marc-schema-with-solr-and-extensions.json
+  printf "%s %s> files generated at marc-schema\n" $(date +"%F %T")
 }
 
 do_all_analyses() {
@@ -326,7 +327,7 @@ commands:
   prepare-solr               prepare indexing
   index                      indexing with Solr
   sqlite                     import tables to SQLite
-  recreate-schema-files      recreate schema files
+  export-schema-files        export schema files
   all-analyses               run all analitical tasks
   all-solr                   run all indexing tasks
   all                        run all tasks

diff --git a/src/main/java/de/gwdg/metadataqa/marc/Utils.java b/src/main/java/de/gwdg/metadataqa/marc/Utils.java
@@ -73,6 +73,7 @@ public static MarcVersion package2version(String packageName) {
       case "nkcrtags":    version = MarcVersion.NKCR;    break;
       case "uvatags":     version = MarcVersion.UVA;     break;
       case "b3kattags":   version = MarcVersion.B3KAT;   break;
+      case "kbrtags":     version = MarcVersion.KBR;   break;
       default:            version = MarcVersion.MARC21;  break;
     }
     return version;

diff --git a/src/main/java/de/gwdg/metadataqa/marc/cli/utils/RecordIterator.java b/src/main/java/de/gwdg/metadataqa/marc/cli/utils/RecordIterator.java
@@ -192,7 +192,7 @@ private Record getNextMarc4jRecord(int i, String lastKnownId, MarcReader reader)
       logger.severe(
         String.format(
           "MARC record parsing problem at record #%d (last known ID: %s): %s",
-          (i +1), lastKnownId, e.getLocalizedMessage()));
+          (i + 1), lastKnownId, e.getLocalizedMessage()));
     } catch (Exception e) {
       logger.log(Level.SEVERE, "start", e);
     }

diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/MarcVersion.java b/src/main/java/de/gwdg/metadataqa/marc/definition/MarcVersion.java
@@ -13,6 +13,7 @@ public enum MarcVersion {
   MARC21NO("MARC21NO", "MARC21 profile for Norwegian public libraries"),
   UVA(     "UVA",      "University of Amsterdam"),
   B3KAT(   "B3KAT",    "B3Kat union catalogue of Bibliotheksverbundes Bayern (BVB) and Kooperativen Bibliotheksverbundes Berlin-Brandenburg (KOBV)"),
+  KBR(     "KBR",      "KBR"),
   ;
 
   String code;

diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/structure/DataFieldDefinition.java b/src/main/java/de/gwdg/metadataqa/marc/definition/structure/DataFieldDefinition.java
@@ -163,6 +163,10 @@ public void putVersionSpecificSubfields(MarcVersion marcVersion,
     versionSpecificSubfields.put(marcVersion, subfieldDefinitions);
   }
 
+  public Map<MarcVersion, List<SubfieldDefinition>> getVersionSpecificSubfields() {
+    return versionSpecificSubfields;
+  }
+
   public boolean hasVersionSpecificSubfields(MarcVersion marcVersion) {
     return versionSpecificSubfields.containsKey(marcVersion);
   }

diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/holdings/Tag866.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/holdings/Tag866.java
@@ -1,10 +1,15 @@
 package de.gwdg.metadataqa.marc.definition.tags.holdings;
 
 import de.gwdg.metadataqa.marc.definition.Cardinality;
+import de.gwdg.metadataqa.marc.definition.MarcVersion;
 import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
 import de.gwdg.metadataqa.marc.definition.structure.Indicator;
 import de.gwdg.metadataqa.marc.definition.general.codelist.HoldingSchemeSourceCodes;
 import de.gwdg.metadataqa.marc.definition.general.parser.LinkageParser;
+import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
+
+import java.util.Arrays;
+
 import static de.gwdg.metadataqa.marc.definition.FRBRFunction.*;
 
 /**
@@ -75,5 +80,11 @@ private void initialize() {
     getSubfield("8")
       .setMqTag("fieldLink")
       .setFrbrFunctions(ManagementIdentify, ManagementProcess);
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag010.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag010.java
@@ -1,8 +1,13 @@
 package de.gwdg.metadataqa.marc.definition.tags.tags01x;
 
 import de.gwdg.metadataqa.marc.definition.Cardinality;
+import de.gwdg.metadataqa.marc.definition.MarcVersion;
 import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
 import de.gwdg.metadataqa.marc.definition.structure.Indicator;
+import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
+
+import java.util.Arrays;
+
 import static de.gwdg.metadataqa.marc.definition.FRBRFunction.*;
 
 /**
@@ -62,5 +67,11 @@ private void initialize() {
       .setMqTag("fieldLink")
       .setFrbrFunctions(ManagementIdentify, ManagementProcess)
       .setCompilanceLevels("O");
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag013.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag013.java
@@ -1,12 +1,17 @@
 package de.gwdg.metadataqa.marc.definition.tags.tags01x;
 
 import de.gwdg.metadataqa.marc.definition.Cardinality;
+import de.gwdg.metadataqa.marc.definition.MarcVersion;
 import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
 import de.gwdg.metadataqa.marc.definition.structure.Indicator;
 import de.gwdg.metadataqa.marc.definition.general.codelist.CountryCodes;
 import de.gwdg.metadataqa.marc.definition.general.codelist.OrganizationCodes;
 import de.gwdg.metadataqa.marc.definition.general.parser.LinkageParser;
 import de.gwdg.metadataqa.marc.definition.general.parser.YYYYMMDDDateParser;
+import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
+
+import java.util.Arrays;
+
 import static de.gwdg.metadataqa.marc.definition.FRBRFunction.*;
 
 /**
@@ -92,5 +97,11 @@ private void initialize() {
     getSubfield("8")
       .setMqTag("fieldLink")
       .setCompilanceLevels("O");
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag015.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag015.java
@@ -1,10 +1,15 @@
 package de.gwdg.metadataqa.marc.definition.tags.tags01x;
 
 import de.gwdg.metadataqa.marc.definition.Cardinality;
+import de.gwdg.metadataqa.marc.definition.MarcVersion;
 import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
 import de.gwdg.metadataqa.marc.definition.structure.Indicator;
 import de.gwdg.metadataqa.marc.definition.general.codelist.NationalBibliographyNumberSourceCodes;
 import de.gwdg.metadataqa.marc.definition.general.parser.LinkageParser;
+import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
+
+import java.util.Arrays;
+
 import static de.gwdg.metadataqa.marc.definition.FRBRFunction.*;
 
 /**
@@ -75,5 +80,11 @@ private void initialize() {
       .setMqTag("fieldLink")
       .setFrbrFunctions(ManagementIdentify, ManagementProcess)
       .setCompilanceLevels("O");
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag016.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag016.java
@@ -1,9 +1,14 @@
 package de.gwdg.metadataqa.marc.definition.tags.tags01x;
 
 import de.gwdg.metadataqa.marc.definition.Cardinality;
+import de.gwdg.metadataqa.marc.definition.MarcVersion;
 import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
 import de.gwdg.metadataqa.marc.definition.structure.Indicator;
 import de.gwdg.metadataqa.marc.definition.general.codelist.OrganizationCodes;
+import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
+
+import java.util.Arrays;
+
 import static de.gwdg.metadataqa.marc.definition.FRBRFunction.*;
 
 /**
@@ -71,5 +76,11 @@ private void initialize() {
       .setMqTag("fieldLink")
       .setFrbrFunctions(ManagementIdentify, ManagementProcess)
       .setCompilanceLevels("O");
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag017.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag017.java
@@ -1,10 +1,15 @@
 package de.gwdg.metadataqa.marc.definition.tags.tags01x;
 
 import de.gwdg.metadataqa.marc.definition.Cardinality;
+import de.gwdg.metadataqa.marc.definition.MarcVersion;
 import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
 import de.gwdg.metadataqa.marc.definition.structure.Indicator;
 import de.gwdg.metadataqa.marc.definition.general.codelist.CopyrightAndLegalDepositNumberSourceCodes;
 import de.gwdg.metadataqa.marc.definition.general.parser.LinkageParser;
+import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
+
+import java.util.Arrays;
+
 import static de.gwdg.metadataqa.marc.definition.FRBRFunction.*;
 
 /**
@@ -99,5 +104,11 @@ private void initialize() {
       .setMqTag("fieldLink")
       .setFrbrFunctions(ManagementIdentify, ManagementProcess)
       .setCompilanceLevels("O");
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag018.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag018.java
@@ -1,9 +1,14 @@
 package de.gwdg.metadataqa.marc.definition.tags.tags01x;
 
 import de.gwdg.metadataqa.marc.definition.Cardinality;
+import de.gwdg.metadataqa.marc.definition.MarcVersion;
 import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
 import de.gwdg.metadataqa.marc.definition.structure.Indicator;
 import de.gwdg.metadataqa.marc.definition.general.parser.LinkageParser;
+import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
+
+import java.util.Arrays;
+
 import static de.gwdg.metadataqa.marc.definition.FRBRFunction.*;
 
 /**
@@ -60,5 +65,11 @@ private void initialize() {
       .setMqTag("fieldLink")
       .setFrbrFunctions(ManagementIdentify, ManagementProcess)
       .setCompilanceLevels("O");
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag020.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag020.java
@@ -90,5 +90,11 @@ private void initialize() {
     putVersionSpecificSubfields(MarcVersion.DNB, Arrays.asList(
       new SubfieldDefinition("9", "ISBN mit Bindestrichen", "R")
     ));
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag022.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag022.java
@@ -1,10 +1,15 @@
 package de.gwdg.metadataqa.marc.definition.tags.tags01x;
 
 import de.gwdg.metadataqa.marc.definition.Cardinality;
+import de.gwdg.metadataqa.marc.definition.MarcVersion;
 import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
 import de.gwdg.metadataqa.marc.definition.structure.Indicator;
 import de.gwdg.metadataqa.marc.definition.general.parser.LinkageParser;
 import de.gwdg.metadataqa.marc.definition.general.validator.ISSNValidator;
+import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
+
+import java.util.Arrays;
+
 import static de.gwdg.metadataqa.marc.definition.FRBRFunction.*;
 
 /**
@@ -106,5 +111,11 @@ private void initialize() {
       "b", "Form of issue [OBSOLETE] [CAN/MARC only]",
       "c", "Price [OBSOLETE] [CAN/MARC only]"
     );
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag024.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag024.java
@@ -123,5 +123,11 @@ private void initialize() {
     putVersionSpecificSubfields(MarcVersion.DNB, Arrays.asList(
       new SubfieldDefinition("9", "Standardnummer (mit Bindestrichen)", "NR")
     ));
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag025.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag025.java
@@ -1,8 +1,13 @@
 package de.gwdg.metadataqa.marc.definition.tags.tags01x;
 
 import de.gwdg.metadataqa.marc.definition.Cardinality;
+import de.gwdg.metadataqa.marc.definition.MarcVersion;
 import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
 import de.gwdg.metadataqa.marc.definition.structure.Indicator;
+import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
+
+import java.util.Arrays;
+
 import static de.gwdg.metadataqa.marc.definition.FRBRFunction.*;
 
 /**
@@ -50,5 +55,11 @@ private void initialize() {
       .setMqTag("fieldLink")
       .setFrbrFunctions(ManagementIdentify, ManagementProcess)
       .setCompilanceLevels("O");
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag026.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag026.java
@@ -1,11 +1,16 @@
 package de.gwdg.metadataqa.marc.definition.tags.tags01x;
 
 import de.gwdg.metadataqa.marc.definition.Cardinality;
+import de.gwdg.metadataqa.marc.definition.MarcVersion;
 import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
 import de.gwdg.metadataqa.marc.definition.structure.Indicator;
 import de.gwdg.metadataqa.marc.definition.general.codelist.FingerprintSchemeSourceCodes;
 import de.gwdg.metadataqa.marc.definition.general.codelist.OrganizationCodes;
 import de.gwdg.metadataqa.marc.definition.general.parser.LinkageParser;
+import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
+
+import java.util.Arrays;
+
 import static de.gwdg.metadataqa.marc.definition.FRBRFunction.*;
 
 /**
@@ -97,5 +102,11 @@ private void initialize() {
       .setMqTag("fieldLink")
       .setFrbrFunctions(ManagementIdentify, ManagementProcess)
       .setCompilanceLevels("O");
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }
diff --git a/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag027.java b/src/main/java/de/gwdg/metadataqa/marc/definition/tags/tags01x/Tag027.java
@@ -1,9 +1,14 @@
 package de.gwdg.metadataqa.marc.definition.tags.tags01x;
 
 import de.gwdg.metadataqa.marc.definition.Cardinality;
+import de.gwdg.metadataqa.marc.definition.MarcVersion;
 import de.gwdg.metadataqa.marc.definition.structure.DataFieldDefinition;
 import de.gwdg.metadataqa.marc.definition.structure.Indicator;
 import de.gwdg.metadataqa.marc.definition.general.parser.LinkageParser;
+import de.gwdg.metadataqa.marc.definition.structure.SubfieldDefinition;
+
+import java.util.Arrays;
+
 import static de.gwdg.metadataqa.marc.definition.FRBRFunction.*;
 
 /**
@@ -69,5 +74,11 @@ private void initialize() {
       .setMqTag("fieldLink")
       .setFrbrFunctions(ManagementIdentify, ManagementProcess)
       .setCompilanceLevels("O");
+
+    putVersionSpecificSubfields(MarcVersion.KBR, Arrays.asList(
+      new SubfieldDefinition("*", "Link with identifier", "NR"),
+      new SubfieldDefinition("@", "Language of field", "NR"),
+      new SubfieldDefinition("#", "number/occurrence of field", "NR")
+    ));
   }
 }