Skip to content

Commit

Permalink
Support custom fetch function (#46)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeswr
jeswr authored Aug 7, 2023
1 parent 2accbad commit 29419ac
Show file tree
Hide file tree
Showing 4 changed files with 107 additions and 4 deletions.
8 changes: 7 additions & 1 deletion lib/RdfDereferencerBase.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {IActorDereferenceRdfOutput} from "@comunica/bus-dereference-rdf";
import { ActionContext, Actor } from "@comunica/core";
import * as RDF from "@rdfjs/types";
import { MediatorDereferenceRdf } from '@comunica/bus-dereference-rdf';
import { KeysHttp } from '@comunica/context-entries';

/**
* An RdfDerefencer can dereference URLs to RDF streams, using any RDF serialization.
Expand All @@ -21,9 +22,10 @@ export class RdfDereferencerBase<Q extends RDF.BaseQuad = RDF.Quad> {
* @return {IActorRdfDereferenceOutput} The dereference output.
*/
public dereference(url: string, options: IDereferenceOptions = {}): Promise<IActorDereferenceRdfOutput> {
const context = new ActionContext(options);
// Delegate dereferencing to the mediator
return this.mediatorDereferenceRdf.mediate({
context: new ActionContext(options),
context: typeof options.fetch === 'function' ? context.setDefault(KeysHttp.fetch, options.fetch) : context,
headers: new Headers(options.headers),
method: options.method,
url,
Expand All @@ -47,6 +49,10 @@ export interface IDereferenceOptions {
* This is not possible in browser environments.
*/
localFiles?: boolean;
/**
* The fetch function to use.
*/
fetch?: typeof fetch;
}

export interface IRdfDerefencerArgs {
Expand Down
2 changes: 2 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
],
"devDependencies": {
"@comunica/runner": "^2.0.3",
"@comunica/types": "^2.8.1",
"@types/jest": "^27.0.0",
"@types/n3": "^1.10.3",
"arrayify-stream": "^1.0.0",
Expand Down Expand Up @@ -116,6 +117,7 @@
"@comunica/bus-rdf-parse": "^2.0.1",
"@comunica/bus-rdf-parse-html": "^2.0.1",
"@comunica/config-query-sparql": "^2.0.1",
"@comunica/context-entries": "^2.8.1",
"@comunica/core": "^2.0.1",
"@comunica/mediator-combine-pipeline": "^2.0.1",
"@comunica/mediator-combine-union": "^2.0.1",
Expand Down
29 changes: 26 additions & 3 deletions test/RdfDereferencer-test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,23 @@ let fetchOut: any;
return fetchOut;
});

const mockSetup = (options: { statusCode?: number, error?: boolean, body?: Readable, headers?: any, url?: string }) => {
const getMock = (options: { statusCode?: number, error?: boolean, body?: Readable, headers?: any, url?: string }) => {
if (options.error) {
fetchOut = Promise.reject(new Error('fetch error'));
return Promise.reject(new Error('fetch error'));
} else {
fetchOut = Promise.resolve({
return Promise.resolve({
url: options.url || 'http://example.org/',
status: options.statusCode || 200,
body: options.body,
headers: new Headers(options.headers || {}),
});
}
};

const mockSetup = (...args: Parameters<typeof getMock>) => {
fetchOut = getMock(...args);
};

mockSetup({ statusCode: 200 });

describe('dereferencer', () => {
Expand Down Expand Up @@ -66,6 +71,24 @@ describe('dereferencer', () => {
]);
});

it('should use custom fetch function when provided', async () => {
const body = new Readable();
body.push(`
<http://ex.org/s> <http://ex.org/p> <http://ex.org/o1>, <http://ex.org/o2>.
`);
body.push(null);

const myFetch: typeof fetch = async (url) => getMock({ statusCode: 200, body, headers: { 'content-type': 'text/turtle' } }) as any;

const out = await dereferencer.dereference('http://example.org/', { fetch: myFetch });
expect(out.metadata.triples).toBeTruthy();
expect(out.url).toEqual('http://example.org/');
return expect(arrayifyStream(out.data)).resolves.toBeRdfIsomorphic([
quad('http://ex.org/s', 'http://ex.org/p', 'http://ex.org/o1'),
quad('http://ex.org/s', 'http://ex.org/p', 'http://ex.org/o2'),
]);
});

it('should handle text/turtle responses by extension', async () => {
const body = new Readable();
body.push(`
Expand Down
72 changes: 72 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -595,6 +595,17 @@
jsonld-context-parser "^2.2.2"
sparqlalgebrajs "^4.0.5"

"@comunica/context-entries@^2.8.1":
version "2.8.1"
resolved "https://registry.yarnpkg.com/@comunica/context-entries/-/context-entries-2.8.1.tgz#7f5af0e7079d93a2bc609e82bc2daec480b96338"
integrity sha512-RjRgAehp9Ujaaqvm/5PJd11l4VHUNqrApV/6ZpjdPbWsJKdMo2mpjyjelEn48n2kid+TXK4a1iKF3lwH+mjRvw==
dependencies:
"@comunica/core" "^2.8.1"
"@comunica/types" "^2.8.1"
"@rdfjs/types" "*"
jsonld-context-parser "^2.2.2"
sparqlalgebrajs "^4.2.0"

"@comunica/core@^2.0.1":
version "2.0.1"
resolved "https://registry.yarnpkg.com/@comunica/core/-/core-2.0.1.tgz#69307bddb182f3402a948d605919c79d2bdbd641"
Expand All @@ -611,6 +622,14 @@
"@comunica/types" "^2.6.0"
immutable "^4.1.0"

"@comunica/core@^2.8.1":
version "2.8.1"
resolved "https://registry.yarnpkg.com/@comunica/core/-/core-2.8.1.tgz#80207da1bf03a49ed088522160f6d11d13663954"
integrity sha512-/rcRdtM0chk6L1ZaZgz84HUbGkFGehR1uAM8Mj0UDfcSZjpV1qUedfF5ktmOA2Er3lz9DZ05Xotgxw6zivILWw==
dependencies:
"@comunica/types" "^2.8.1"
immutable "^4.1.0"

"@comunica/mediator-combine-pipeline@^2.0.1":
version "2.0.1"
resolved "https://registry.yarnpkg.com/@comunica/mediator-combine-pipeline/-/mediator-combine-pipeline-2.0.1.tgz#afbd88b31405b39193807264ff1bf3b341e688ac"
Expand Down Expand Up @@ -675,6 +694,16 @@
asynciterator "^3.8.0"
sparqlalgebrajs "^4.0.5"

"@comunica/types@^2.8.1":
version "2.8.1"
resolved "https://registry.yarnpkg.com/@comunica/types/-/types-2.8.1.tgz#f0e81b78ac8767eddc9f03b2342389ff21ec959b"
integrity sha512-uRBKBYpG1RVuQw8hfHHE1jdiCt6FTDeM2r4YqFY2OSOfxjrIDBYkcqsgod1m5sFs3l/P6CVVZG9w6uZ7876Y8w==
dependencies:
"@rdfjs/types" "*"
"@types/yargs" "^17.0.13"
asynciterator "^3.8.1"
sparqlalgebrajs "^4.2.0"

"@dabh/diagnostics@^2.0.2":
version "2.0.3"
resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.3.tgz#7f7e97ee9a725dffc7808d93668cc984e1dc477a"
Expand Down Expand Up @@ -1458,6 +1487,11 @@ asynciterator@^3.8.0:
resolved "https://registry.yarnpkg.com/asynciterator/-/asynciterator-3.8.0.tgz#d9246a3d39ce2a9e4b020834d7f352adf35b49d8"
integrity sha512-bD34LqKHJnkB77MHjL3hOAUOcy9dbB+3lHvL+EiJpD3k2Nyq3i1dCk5adMisB2rwlrHVu/+XRhOdPZL9hzpsfw==

asynciterator@^3.8.1:
version "3.8.1"
resolved "https://registry.yarnpkg.com/asynciterator/-/asynciterator-3.8.1.tgz#80be735b252332494e186ee733544e5b21dd2123"
integrity sha512-SmdG0FUY3pYGOZZGdYq8Qb/DCRDXBFZUk08V1/4lbBXdAQvcC3Kxzz9FUDPBTik7VAVltt4cZirAPtJv3gOpEw==

asynckit@^0.4.0:
version "0.4.0"
resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
Expand Down Expand Up @@ -3851,6 +3885,13 @@ rdf-data-factory@^1.1.1:
dependencies:
"@rdfjs/types" "*"

rdf-data-factory@^1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/rdf-data-factory/-/rdf-data-factory-1.1.2.tgz#d47550d2649d0d64f8cae3fcc9efae7a8a895d9a"
integrity sha512-TfQD63Lokabd09ES1jAtKK8AA6rkr9rwyUBGo6olOt1CE0Um36CUQIqytyf0am2ouBPR0l7SaHxCiMcPGHkt1A==
dependencies:
"@rdfjs/types" "*"

rdf-isomorphic@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/rdf-isomorphic/-/rdf-isomorphic-1.3.0.tgz#6a1ad38b27c5ebed8a5f752b15f81988f48f8e22"
Expand Down Expand Up @@ -3932,6 +3973,15 @@ rdf-string@^1.5.0, rdf-string@^1.6.0:
"@rdfjs/types" "*"
rdf-data-factory "^1.1.0"

rdf-terms@^1.10.0:
version "1.11.0"
resolved "https://registry.yarnpkg.com/rdf-terms/-/rdf-terms-1.11.0.tgz#0c2e3a2b43f1042959c9263af27dab08dc4b084d"
integrity sha512-iKlVgnMopRKl9pHVNrQrax7PtZKRCT/uJIgYqvuw1VVQb88zDvurtDr1xp0rt7N9JtKtFwUXoIQoEsjyRo20qQ==
dependencies:
"@rdfjs/types" "*"
rdf-data-factory "^1.1.0"
rdf-string "^1.6.0"

rdf-terms@^1.7.0:
version "1.7.1"
resolved "https://registry.yarnpkg.com/rdf-terms/-/rdf-terms-1.7.1.tgz#d53891b208b357ed4b7738134decd537130098e6"
Expand Down Expand Up @@ -4301,6 +4351,21 @@ sparqlalgebrajs@^4.0.5:
rdf-string "^1.6.0"
sparqljs "^3.6.1"

sparqlalgebrajs@^4.2.0:
version "4.2.0"
resolved "https://registry.yarnpkg.com/sparqlalgebrajs/-/sparqlalgebrajs-4.2.0.tgz#b70945bdd2ab2897a0bc22adb804dd1a2b0ec98c"
integrity sha512-tdlJdrvgQqgx9zubcl9iiyCxMOp4qRT2fs1Sne8X35QTm1Pj2ulB+gGEHunJJnw5FW7Uhtmw7J3px0sCmgJSbw==
dependencies:
"@rdfjs/types" "*"
"@types/sparqljs" "^3.1.3"
fast-deep-equal "^3.1.3"
minimist "^1.2.6"
rdf-data-factory "^1.1.0"
rdf-isomorphic "^1.3.0"
rdf-string "^1.6.0"
rdf-terms "^1.10.0"
sparqljs "^3.7.1"

sparqljs@^3.5.1:
version "3.5.1"
resolved "https://registry.yarnpkg.com/sparqljs/-/sparqljs-3.5.1.tgz#631cd4b5bcc3d9ce23c5313b3a158eccf858e28b"
Expand All @@ -4315,6 +4380,13 @@ sparqljs@^3.6.1:
dependencies:
rdf-data-factory "^1.1.1"

sparqljs@^3.7.1:
version "3.7.1"
resolved "https://registry.yarnpkg.com/sparqljs/-/sparqljs-3.7.1.tgz#5d121895d491d50214f2e38f2885a3a935b6c093"
integrity sha512-I1jYMtcwDkgCEqQ4eQuQIhB8hFAlRAJ6YDXDcV54XztaJaYRFqJlidHt77S3j8Mfh6kY6GK04dXPEIopxbEeuQ==
dependencies:
rdf-data-factory "^1.1.2"

spawn-sync@^1.0.15:
version "1.0.15"
resolved "https://registry.yarnpkg.com/spawn-sync/-/spawn-sync-1.0.15.tgz#b00799557eb7fb0c8376c29d44e8a1ea67e57476"
Expand Down

0 comments on commit 29419ac

Please sign in to comment.