-
Notifications
You must be signed in to change notification settings - Fork 2
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Make CheckUrlExists work on Node18 #157
Changes from all commits
bdb3ae1
de93919
053024a
7c6ff1a
fc90e60
238e4eb
2c4f632
be985ba
8c98c6b
7512599
97ba06d
ca18a4c
7b23190
78b2b93
09e118e
a462862
4522218
f6d5816
e753134
17eea92
6f1e84a
ee26337
50c9e13
9b10bb5
2e4fbe8
f986208
482bb15
1980520
df7ae20
1035435
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,22 +1,12 @@ | ||
const fs = require("fs"); | ||
const tls = require("tls"); | ||
const Url = require("url"); | ||
const ftp = require("basic-ftp"); | ||
const { http, https } = require("follow-redirects"); | ||
|
||
const { curly } = require("node-libcurl"); | ||
|
||
// important steps to get validation of https (as opposed to http) urls | ||
// Get root certificates so https will work | ||
// | ||
// Write the certificates to a file | ||
// https://stackoverflow.com/questions/63052127/protractor-node-libcurl-failed-ssl-peer-certificate-or-ssh-remote-key-was-not-o | ||
// When doing sam build the file must be in /tmp because other wise it cannot be read | ||
// due to ro file system in container | ||
// https://stackoverflow.com/questions/53810516/getting-error-aws-lambda-erofs-read-only-file-system-open-var-task-assets | ||
const certFilePath = "/tmp/cacert.pem"; | ||
// https://nodejs.org/api/tls.html#tls_tls_rootcertificates | ||
// An immutable array of strings representing the root certificates (in PEM format) from the bundled Mozilla CA store as supplied by current Node.js version. | ||
// The bundled CA store, as supplied by Node.js, is a snapshot of Mozilla CA store that is fixed at release time. It is identical on all supported platforms. | ||
const tlsData = tls.rootCertificates.join("\n"); | ||
fs.writeFileSync(certFilePath, tlsData); | ||
// The Node url.parse returns an object where the protocol is lower case and contains the colon at the end | ||
const SECURE_FTP_PROTOCOL = "sftp:"; | ||
const FTP_PROTOCOL = "ftp:"; | ||
const HTTP_PROTOCOL = "http:"; | ||
const HTTPS_PROTOCOL = "https:"; | ||
|
||
/** | ||
* TODO: Change to array of URLs to parse | ||
|
@@ -50,10 +40,48 @@ async function checkUrl(url) { | |
} | ||
|
||
async function run(url) { | ||
const curlOpts = { | ||
caInfo: certFilePath, | ||
}; | ||
return curly.head(url, curlOpts); | ||
const parsedUrl = Url.parse(url); | ||
const protocol = parsedUrl.protocol; // Url.parse() lower cases the protocol | ||
if (FTP_PROTOCOL === protocol || SECURE_FTP_PROTOCOL === protocol) { | ||
const secure = SECURE_FTP_PROTOCOL === protocol; | ||
const ftpClient = new ftp.Client(); | ||
try { | ||
let options = { | ||
host: parsedUrl.host, | ||
secure: secure, | ||
...(parsedUrl.port && { port: parsedUrl.port }), | ||
}; | ||
await ftpClient.access(options); | ||
const size = await ftpClient.size(parsedUrl.path); | ||
return size > 0 | ||
? Promise.resolve() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is it plausible that a valid file will be empty? If so, There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmm, I was thinking an empty file would never be valid from a workflow input perspective. But I could see it both ways. I'm going to leave it as is for now, but if you or anybody feels strongly, I can change it. |
||
: Promise.reject("Could not get size for " + url); | ||
} finally { | ||
ftpClient.close(); | ||
} | ||
} else if (HTTP_PROTOCOL === protocol) { | ||
return httpOrHttpsRequest(url, http); | ||
} else if (HTTPS_PROTOCOL === protocol) { | ||
return httpOrHttpsRequest(url, https); | ||
} | ||
return Promise.reject("Unsupported protocol: " + protocol); | ||
} | ||
|
||
function httpOrHttpsRequest(url, httpOrHttps) { | ||
return new Promise((resolve, reject) => { | ||
const req = httpOrHttps.request(url, { | ||
method: "HEAD", | ||
headers: { "user-agent": "Dockstore/1.0" }, // User-agent must be set for tests to pass, AWS (WAF?) blocks requests with no user-agent | ||
}); | ||
req.on("response", (res) => { | ||
if (res.statusCode < 300) { | ||
resolve(res.statusCode); | ||
} | ||
reject(res.statusCode); | ||
}); | ||
req.on("error", (err) => reject(err)); | ||
req.end(); | ||
}); | ||
} | ||
|
||
function returnResponse(fileFound) { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If this or any other
await
fails, it will throw an exception, which will be caught on line 46/36 (note to self and anybody else who isn't familiar with JS promises andawait
.