Skip to content

Commit

Permalink
feat: Add license text as evidence (#1243)
Browse files Browse the repository at this point in the history
fixes #256 

This PR is based on #427, and uses the same implementation as it was
implemented in yarn (See
CycloneDX/cyclonedx-node-yarn#193)
closes #427

---------

Signed-off-by: Matthias Schiebel <msl@compart.com>
Signed-off-by: Christoph Uhland <42832096+cuhland@users.noreply.github.com>
Signed-off-by: Christoph Uhland <christoph.uhland@bdr.de>
Co-authored-by: Matthias Schiebel <msl@compart.com>
  • Loading branch information
cuhland and Matthias Schiebel authored Dec 17, 2024
1 parent 04c8d4d commit 0c7e2d4
Show file tree
Hide file tree
Showing 6 changed files with 215 additions and 4 deletions.
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ Options:
--output-reproducible Whether to go the extra mile and make the output reproducible.
This requires more resources, and might result in loss of time- and random-based-values.
(env: BOM_REPRODUCIBLE)
--gather-license-texts Search for license files in components and include them as license evidence.
This feature is experimental. (default: false)
--output-format <format> Which output format to use.
(choices: "JSON", "XML", default: "JSON")
--output-file <file> Path to the output file.
Expand Down
23 changes: 23 additions & 0 deletions src/_helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ Copyright (c) OWASP Foundation. All Rights Reserved.
*/

import { readFileSync, writeSync } from 'fs'
import { parse } from 'path'

export function loadJsonFile (path: string): any {
return JSON.parse(readFileSync(path, 'utf8'))
Expand Down Expand Up @@ -56,3 +57,25 @@ export function tryRemoveSecretsFromUrl (url: string): string {
return url
}
}

export const LICENSE_FILENAME_PATTERN = /^(?:UN)?LICEN[CS]E|.\.LICEN[CS]E$|^NOTICE$/i
const LICENSE_FILENAME_BASE = new Set(['licence', 'license'])
const LICENSE_FILENAME_EXT = new Set(['.apache', '.bsd', '.gpl', '.mit'])
const MAP_TEXT_EXTENSION_MIME = new Map([
['', 'text/plain'],
['.htm', 'text/html'],
['.html', 'text/html'],
['.md', 'text/markdown'],
['.txt', 'text/plain'],
['.rst', 'text/prs.fallenstein.rst'],
['.xml', 'text/xml'],
['.license', 'text/plain'],
['.licence', 'text/plain']
])

export function getMimeForLicenseFile (filename: string): string | undefined {
const { name, ext } = parse(filename.toLowerCase())
return LICENSE_FILENAME_BASE.has(name) && LICENSE_FILENAME_EXT.has(ext)
? 'text/plain'
: MAP_TEXT_EXTENSION_MIME.get(ext)
}
59 changes: 56 additions & 3 deletions src/builders.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,17 @@ Copyright (c) OWASP Foundation. All Rights Reserved.
*/

import { type Builders, Enums, type Factories, Models, Utils } from '@cyclonedx/cyclonedx-library'
import { existsSync } from 'fs'
import { existsSync, readdirSync, readFileSync } from 'fs'
import * as normalizePackageData from 'normalize-package-data'
import * as path from 'path'

import { isString, loadJsonFile, tryRemoveSecretsFromUrl } from './_helpers'
import { join } from 'path'

import {
getMimeForLicenseFile,
isString,
LICENSE_FILENAME_PATTERN,
loadJsonFile, tryRemoveSecretsFromUrl
} from './_helpers'
import { makeNpmRunner, type runFunc } from './npmRunner'
import { PropertyNames, PropertyValueBool } from './properties'
import { versionCompare } from './versionCompare'
Expand All @@ -37,6 +43,7 @@ interface BomBuilderOptions {
reproducible?: BomBuilder['reproducible']
flattenComponents?: BomBuilder['flattenComponents']
shortPURLs?: BomBuilder['shortPURLs']
gatherLicenseTexts?: BomBuilder['gatherLicenseTexts']
}

type cPath = string
Expand All @@ -56,6 +63,7 @@ export class BomBuilder {
reproducible: boolean
flattenComponents: boolean
shortPURLs: boolean
gatherLicenseTexts: boolean

console: Console

Expand All @@ -79,6 +87,7 @@ export class BomBuilder {
this.reproducible = options.reproducible ?? false
this.flattenComponents = options.flattenComponents ?? false
this.shortPURLs = options.shortPURLs ?? false
this.gatherLicenseTexts = options.gatherLicenseTexts ?? false

this.console = console_
}
Expand Down Expand Up @@ -465,6 +474,23 @@ export class BomBuilder {
l.acknowledgement = Enums.LicenseAcknowledgement.Declared
})

if (this.gatherLicenseTexts) {
if (this.packageLockOnly) {
this.console.warn('WARN | Adding license text is ignored (package-lock-only is configured!) for %j', data.name)
} else {
component.evidence = new Models.ComponentEvidence()
for (const license of this.fetchLicenseEvidence(data?.path as string)) {
if (license != null) {
// only create a evidence if a license attachment is found
if (component.evidence == null) {
component.evidence = new Models.ComponentEvidence()
}
component.evidence.licenses.add(license)
}
}
}
}

if (isOptional || isDevOptional) {
component.scope = Enums.ComponentScope.Optional
}
Expand Down Expand Up @@ -610,6 +636,33 @@ export class BomBuilder {
}
}
}

private * fetchLicenseEvidence (path: string): Generator<Models.License | null, void, void> {
const files = readdirSync(path)
for (const file of files) {
if (!LICENSE_FILENAME_PATTERN.test(file)) {
continue
}

const contentType = getMimeForLicenseFile(file)
if (contentType === undefined) {
continue
}

const fp = join(path, file)
yield new Models.NamedLicense(
`file: ${file}`,
{
text: new Models.Attachment(
readFileSync(fp).toString('base64'),
{
contentType,
encoding: Enums.AttachmentEncoding.Base64
}
)
})
}
}
}

class DummyComponent extends Models.Component {
Expand Down
10 changes: 9 additions & 1 deletion src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ interface CommandOptions {
flattenComponents: boolean
shortPURLs: boolean
outputReproducible: boolean
gatherLicenseTexts: boolean
outputFormat: OutputFormat
outputFile: string
validate: boolean | undefined
Expand Down Expand Up @@ -115,6 +116,12 @@ function makeCommand (process: NodeJS.Process): Command {
).env(
'BOM_REPRODUCIBLE'
)
).addOption(
new Option(
'--gather-license-texts',
'Search for license files in components and include them as license evidence.\n' +
'This feature is experimental. (default: false)'
).default(false)
).addOption(
(function () {
const o = new Option(
Expand Down Expand Up @@ -249,7 +256,8 @@ export async function run (process: NodeJS.Process): Promise<number> {
omitDependencyTypes: options.omit,
reproducible: options.outputReproducible,
flattenComponents: options.flattenComponents,
shortPURLs: options.shortPURLs
shortPURLs: options.shortPURLs,
gatherLicenseTexts: options.gatherLicenseTexts
},
myConsole
).buildFromProjectDir(projectDir, process)
Expand Down
56 changes: 56 additions & 0 deletions tests/unit/_helpers.spec.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*!
This file is part of CycloneDX generator for NPM projects.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
SPDX-License-Identifier: Apache-2.0
Copyright (c) OWASP Foundation. All Rights Reserved.
*/

const { getMimeForLicenseFile, LICENSE_FILENAME_PATTERN } = require('../../dist/_helpers')
const { describe, expect, test } = require('@jest/globals')

describe('LICENSE_FILENAME_PATTERN', () => {
test.each([
'LICENCE',
'licence',
'LICENSE',
'license',
'NOTICE',
'UNLICENCE',
'UNLICENSE'])('valid name: %s', (fileName) => {
const value = LICENSE_FILENAME_PATTERN.test(fileName)
expect(value).toBeTruthy()
})
test.each([
'my-license',
'my_license',
'myNotice',
'the-LICENSE'])('invalid name: %s', (fileName) => {
const value = LICENSE_FILENAME_PATTERN.test(fileName)
expect(value).toBeFalsy()
})
})

describe('getMimeForLicenseFile', () => {
test.each([
['LICENCE', 'text/plain'],
['site.html', 'text/html'],
['license.md', 'text/markdown'],
['info.xml', 'text/xml'],
['UNKNOWN', 'text/plain']
])('check %s', (fileName, expected) => {
const value = getMimeForLicenseFile(fileName)
expect(value).toBe(expected)
})
})
69 changes: 69 additions & 0 deletions tests/unit/builders.BomBuilder.spec.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*!
This file is part of CycloneDX generator for NPM projects.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
SPDX-License-Identifier: Apache-2.0
Copyright (c) OWASP Foundation. All Rights Reserved.
*/

const { Models, Builders, Factories } = require('@cyclonedx/cyclonedx-library')
const fs = require('fs')
const { BomBuilder, TreeBuilder } = require('../../dist/builders')
const { jest, describe, expect, it } = require('@jest/globals')

describe('BomBuilder', () => {
const extRefFactory = new Factories.FromNodePackageJson.ExternalReferenceFactory()
const bomBuilder = new BomBuilder(
new Builders.FromNodePackageJson.ToolBuilder(extRefFactory),
new Builders.FromNodePackageJson.ComponentBuilder(
extRefFactory,
new Factories.LicenseFactory()
),
new TreeBuilder(),
new Factories.FromNodePackageJson.PackageUrlFactory('npm'),
{
ignoreNpmErrors: true,
metaComponentType: true,
packageLockOnly: true,
omitDependencyTypes: [],
reproducible: true,
flattenComponents: true,
shortPURLs: true,
gatherLicenseTexts: true
},
null
)
describe('License fetching', () => {
it('fetches existing license from directory', async () => {
const fsMock = jest.spyOn(fs, 'readdirSync')
const fileMock = jest.spyOn(fs, 'readFileSync')
fsMock.mockReturnValue(['license.txt'])
fileMock.mockReturnValue('license file content')
const licenses = bomBuilder.fetchLicenseEvidence('test_module')
const license = licenses.next().value
expect(license).toBeInstanceOf(Models.NamedLicense)
expect(license.name).toBe('file: license.txt')
expect(license.text.contentType).toBe('text/plain')
expect(license.text.encoding).toBe('base64')
expect(license.text.content).toBe('license file content')
})
it('fetches nothing from directory', async () => {
const fsMock = jest.spyOn(fs, 'readdirSync')
fsMock.mockReturnValue(['nothing.txt'])
const licenses = bomBuilder.fetchLicenseEvidence('test_module')
const license = licenses.next().value
expect(license).toBeFalsy()
})
})
})

0 comments on commit 0c7e2d4

Please sign in to comment.