Add option to normalize language tags in 1.1

rubensworks · Mar 16, 2020 · 1b1e95f · 1b1e95f
1 parent 0b7ae17
commit 1b1e95f
Show file tree

Hide file tree

Showing 7 changed files with 69 additions and 9 deletions.
diff --git a/lib/JsonLdParser.ts b/lib/JsonLdParser.ts
@@ -459,4 +459,10 @@ export interface IJsonLdParserOptions {
    * * 'compound-literal': reified values using rdf:value, rdf:direction and rdf:language.
    */
   rdfDirection?: 'i18n-datatype' | 'compound-literal';
+  /**
+   * If language tags should be normalized to lowercase.
+   * This is always true for JSON-LD 1.0,
+   * but false by default for all following versions.
+   */
+  normalizeLanguageTags?: boolean;
 }
diff --git a/lib/ParsingContext.ts b/lib/ParsingContext.ts
@@ -33,6 +33,7 @@ export class ParsingContext {
   public readonly rootContext: Promise<IJsonLdContextNormalized>;
   public readonly defaultGraph?: RDF.Term;
   public readonly rdfDirection?: 'i18n-datatype' | 'compound-literal';
+  public readonly normalizeLanguageTags?: boolean;
 
   // Stack of indicating if a depth has been touched.
   public readonly processingStack: boolean[];
@@ -79,6 +80,7 @@ export class ParsingContext {
     this.strictRanges = options.strictRanges;
     this.defaultGraph = options.defaultGraph;
     this.rdfDirection = options.rdfDirection;
+    this.normalizeLanguageTags = options.normalizeLanguageTags;
 
     // Initialize stacks
     this.processingStack = [];
@@ -95,7 +97,10 @@ export class ParsingContext {
 
     this.parser = options.parser;
     if (options.context) {
-      this.rootContext = this.contextParser.parse(options.context, { baseIri: options.baseIRI });
+      this.rootContext = this.contextParser.parse(options.context, {
+        baseIri: options.baseIRI,
+        normalizeLanguageTags: this.normalizeLanguageTags,
+      });
       this.rootContext.then((context) => this.validateContext(context));
     } else {
       this.rootContext = Promise.resolve(this.baseIRI ? { '@base': this.baseIRI } : {});

diff --git a/lib/Util.ts b/lib/Util.ts
@@ -247,6 +247,11 @@ export class Util {
           if (!ContextParser.validateLanguage(valueLanguage, this.parsingContext.strictRanges)) {
             return null;
           }
+
+          // Language tags are always normalized to lowercase in 1.0.
+          if (this.parsingContext.normalizeLanguageTags || this.parsingContext.activeProcessingMode === 1.0) {
+            valueLanguage = valueLanguage.toLowerCase();
+          }
         }
         if (valueDirection) {
           if (typeof val !== 'string') {
@@ -272,11 +277,6 @@ export class Util {
             throw new Error(`Can not have both '@language' and '@type' in a value: '${JSON.stringify(value)}'`);
           }
 
-          // Language tags are always normalized to lowercase in 1.0.
-          if (this.parsingContext.activeProcessingMode === 1.0) {
-            valueLanguage = valueLanguage.toLowerCase();
-          }
-
           return this.dataFactory.literal(val, valueLanguage);
         } else if (valueDirection && this.parsingContext.rdfDirection) { // Check @direction
           if (valueType) {

diff --git a/lib/entryhandler/keyword/EntryHandlerKeywordContext.ts b/lib/entryhandler/keyword/EntryHandlerKeywordContext.ts
@@ -24,8 +24,11 @@ export class EntryHandlerKeywordContext extends EntryHandlerKeyword {
     const parentContext: Promise<IJsonLdContextNormalized> = parsingContext.getContext(keys.slice(0, -1));
 
     // Set the context for this scope
-    const context = parsingContext.contextParser.parse(value,
-      { baseIri: parsingContext.baseIRI, parentContext: await parentContext });
+    const context = parsingContext.contextParser.parse(value, {
+      baseIri: parsingContext.baseIRI,
+      normalizeLanguageTags: parsingContext.normalizeLanguageTags,
+      parentContext: await parentContext,
+    });
     parsingContext.contextTree.setContext(keys.slice(0, -1), context);
     parsingContext.emitContext(value);
     await parsingContext.validateContext(await context);

diff --git a/spec/parser.js b/spec/parser.js
@@ -12,6 +12,11 @@ module.exports = {
         new ErrorSkipped(`Test with spec version ${options.specVersion} was skipped, only 1.0 is supported.`));
     }
     return require('arrayify-stream')(require('streamify-string')(data)
-      .pipe(new JsonLdParser(Object.assign({ baseIRI, allowOutOfOrderContext: true, validateValueIndexes: true }, options))));
+      .pipe(new JsonLdParser(Object.assign({
+        baseIRI,
+        allowOutOfOrderContext: true,
+        validateValueIndexes: true,
+        normalizeLanguageTags: true, // To simplify testing
+      }, options))));
   },
 };
diff --git a/test/JsonLdParser-test.ts b/test/JsonLdParser-test.ts
@@ -365,6 +365,39 @@ describe('JsonLdParser', () => {
           ]);
         });
 
+        it('with @id and a mixed-case context-language literal', async () => {
+          const stream = streamifyString(`
+{
+  "@context": {
+    "ex": "http://ex.org/",
+    "p": { "@id": "http://ex.org/pred1", "@language": "en-US" }
+  },
+  "@id": "http://ex.org/myid",
+  "p": "my value"
+}`);
+          return expect(await arrayifyStream(stream.pipe(parser))).toBeRdfIsomorphic([
+            triple(namedNode('http://ex.org/myid'), namedNode('http://ex.org/pred1'),
+              literal('my value', 'en-US')),
+          ]);
+        });
+
+        it('with @id and a mixed-case context-language literal when normalizeLanguageTags is true', async () => {
+          const stream = streamifyString(`
+{
+  "@context": {
+    "ex": "http://ex.org/",
+    "p": { "@id": "http://ex.org/pred1", "@language": "en-US" }
+  },
+  "@id": "http://ex.org/myid",
+  "p": "my value"
+}`);
+          parser = new JsonLdParser({ dataFactory, allowOutOfOrderContext, normalizeLanguageTags: true });
+          return expect(await arrayifyStream(stream.pipe(parser))).toBeRdfIsomorphic([
+            triple(namedNode('http://ex.org/myid'), namedNode('http://ex.org/pred1'),
+              literal('my value', 'en-us')),
+          ]);
+        });
+
         it('with @id and literal with default language', async () => {
           const stream = streamifyString(`
 {

diff --git a/test/Util-test.ts b/test/Util-test.ts
@@ -286,6 +286,14 @@ describe('Util', () => {
               .toEqualRdfTerm(literal('abc', 'en-US'));
           });
 
+        it('with an @value and @language should return a lowercased language-tagged string literal' +
+          'if normalizeLanguageTags',
+          async () => {
+            util.parsingContext.normalizeLanguageTags = true;
+            return expect(await util.valueToTerm(context, 'key', { '@value': 'abc', '@language': 'en-US' }, 0))
+              .toEqualRdfTerm(literal('abc', 'en-us'));
+          });
+
         it('with an @value and @type should return a typed literal', async () => {
           return expect(await util.valueToTerm(context, 'key', { '@value': 'abc', '@type': 'http://type.com' }, 0))
             .toEqualRdfTerm(literal('abc', namedNode('http://type.com')));