feat: BREAKING CHANGE - relocated concrete parsers (#299)

* feat: BREAKING CHANGE - reloated concrete parsers from `cyclonedx-python-lib`

Signed-off-by: Paul Horton <phorton@sonatype.com>

* re-located tests for Utils

Signed-off-by: Paul Horton <phorton@sonatype.com>

* feat: BREAKING CHANGE - relocated concreate parsers from `cyclonedx-python-lib`
doc: updated to reflect breaking changes
dod: added changelog

Signed-off-by: Paul Horton <phorton@sonatype.com>

cyclonedx_py/client.py

@@ -27,11 +27,12 @@
 from cyclonedx.model.bom import Bom
 from cyclonedx.output import BaseOutput, get_instance, OutputFormat, SchemaVersion
 from cyclonedx.parser import BaseParser
-from cyclonedx.parser.conda import CondaListExplicitParser, CondaListJsonParser
-from cyclonedx.parser.environment import EnvironmentParser
-from cyclonedx.parser.pipenv import PipEnvParser
-from cyclonedx.parser.poetry import PoetryParser
-from cyclonedx.parser.requirements import RequirementsParser
+
+from .parser.conda import CondaListExplicitParser, CondaListJsonParser
+from .parser.environment import EnvironmentParser
+from .parser.pipenv import PipEnvParser
+from .parser.poetry import PoetryParser
+from .parser.requirements import RequirementsParser
 
 
 class CycloneDxCmdException(Exception):
@@ -67,7 +68,7 @@ def get_output(self) -> BaseOutput:
             print(f'ERROR: {str(e)}')
             exit(1)
 
-        if parser.has_warnings():
+        if parser and parser.has_warnings():
             print('')
             print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
             print('!! Some of your dependencies do not have pinned version !!')

cyclonedx_py/exception/__init__.py

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+"""
+Exceptions that are specific to the CycloneDX Python implementation.
+"""

cyclonedx_py/exception/parser.py

@@ -0,0 +1,29 @@
+# encoding: utf-8
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+"""
+Exceptions that are specific error scenarios during occuring within Parsers in the CycloneDX library implementation.
+"""
+
+from cyclonedx.exception import CycloneDxException
+
+
+class UnknownHashTypeException(CycloneDxException):
+    """
+    Exception raised when we are unable to determine the type of hash from a composite hash string.
+    """
+    pass

cyclonedx_py/parser/__init__.py

@@ -0,0 +1,22 @@
+# encoding: utf-8
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Set of concrete classes and methods which allow for quick creation of a Bom instance from your environment or Python
+project.
+
+Use a Parser instead of programmatically creating a Bom as a developer.
+"""

cyclonedx_py/parser/conda.py

@@ -0,0 +1,97 @@
+# encoding: utf-8
+
+# This file is part of CycloneDX Python Lib
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+
+import json
+from abc import ABCMeta, abstractmethod
+from typing import List
+
+from cyclonedx.model import ExternalReference, ExternalReferenceType
+from cyclonedx.model.component import Component
+from cyclonedx.parser import BaseParser
+
+from ..utils.conda import parse_conda_json_to_conda_package, parse_conda_list_str_to_conda_package, CondaPackage
+
+
+class _BaseCondaParser(BaseParser, metaclass=ABCMeta):
+    """Internal abstract parser - not for programmatic use.
+    """
+
+    def __init__(self, conda_data: str) -> None:
+        super().__init__()
+        self._conda_packages: List[CondaPackage] = []
+        self._parse_to_conda_packages(data_str=conda_data)
+        self._conda_packages_to_components()
+
+    @abstractmethod
+    def _parse_to_conda_packages(self, data_str: str) -> None:
+        """
+        Abstract method for implementation by concrete Conda Parsers.
+
+        Implementation should add a `list` of `CondaPackage` instances to `self._conda_packages`
+
+        Params:
+            data_str:
+                `str` data passed into the Parser
+        """
+        pass
+
+    def _conda_packages_to_components(self) -> None:
+        """
+        Converts the parsed `CondaPackage` instances into `Component` instances.
+
+        """
+        for conda_package in self._conda_packages:
+            c = Component(
+                name=conda_package['name'], version=str(conda_package['version'])
+            )
+            c.add_external_reference(ExternalReference(
+                reference_type=ExternalReferenceType.DISTRIBUTION,
+                url=conda_package['base_url'],
+                comment=f"Distribution name {conda_package['dist_name']}"
+            ))
+
+            self._components.append(c)
+
+
+class CondaListJsonParser(_BaseCondaParser):
+    """
+    This parser is intended to receive the output from the command `conda list --json`.
+    """
+
+    def _parse_to_conda_packages(self, data_str: str) -> None:
+        conda_list_content = json.loads(data_str)
+
+        for package in conda_list_content:
+            conda_package = parse_conda_json_to_conda_package(conda_json_str=json.dumps(package))
+            if conda_package:
+                self._conda_packages.append(conda_package)
+
+
+class CondaListExplicitParser(_BaseCondaParser):
+    """
+    This parser is intended to receive the output from the command `conda list --explicit` or
+    `conda list --explicit --md5`.
+    """
+
+    def _parse_to_conda_packages(self, data_str: str) -> None:
+        for line in data_str.replace('\r\n', '\n').split('\n'):
+            line = line.strip()
+            conda_package = parse_conda_list_str_to_conda_package(conda_list_str=line)
+            if conda_package:
+                self._conda_packages.append(conda_package)

cyclonedx_py/parser/environment.py

@@ -0,0 +1,84 @@
+# encoding: utf-8
+
+# This file is part of CycloneDX Python Lib
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+
+"""
+Parser classes for reading installed packages in your current Python environment.
+
+These parsers look at installed packages only - not what you have defined in any dependency tool - see the other Parsers
+if you want to derive CycloneDX from declared dependencies.
+
+
+The Environment Parsers support population of the following data about Components:
+
+"""
+
+import sys
+
+from pkg_resources import DistInfoDistribution  # type: ignore
+
+if sys.version_info >= (3, 8):
+    from importlib.metadata import metadata
+    from email.message import Message as _MetadataReturn
+else:
+    from importlib_metadata import metadata, PackageMetadata as _MetadataReturn
+
+from cyclonedx.model import LicenseChoice
+from cyclonedx.model.component import Component
+from cyclonedx.parser import BaseParser
+
+
+class EnvironmentParser(BaseParser):
+    """
+    This will look at the current Python environment and list out all installed packages.
+
+    Best used when you have virtual Python environments per project.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+
+        import pkg_resources
+
+        i: DistInfoDistribution
+        for i in iter(pkg_resources.working_set):
+            c = Component(name=i.project_name, version=i.version)
+
+            i_metadata = self._get_metadata_for_package(i.project_name)
+            if 'Author' in i_metadata:
+                c.author = i_metadata['Author']
+
+            if 'License' in i_metadata and i_metadata['License'] != 'UNKNOWN':
+                c.licenses.append(
+                    LicenseChoice(license_expression=i_metadata['License'])
+                )
+
+            if 'Classifier' in i_metadata:
+                for classifier in i_metadata['Classifier']:
+                    if str(classifier).startswith('License :: OSI Approved :: '):
+                        c.licenses.append(
+                            LicenseChoice(
+                                license_expression=str(classifier).replace('License :: OSI Approved :: ', '').strip()
+                            )
+                        )
+
+            self._components.append(c)
+
+    @staticmethod
+    def _get_metadata_for_package(package_name: str) -> _MetadataReturn:
+        return metadata(package_name)

cyclonedx_py/parser/pipenv.py

@@ -0,0 +1,59 @@
+# encoding: utf-8
+
+# This file is part of CycloneDX Python Lib
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+
+import json
+from typing import Any, Dict
+
+from cyclonedx.model import ExternalReference, ExternalReferenceType, HashType
+from cyclonedx.model.component import Component
+from cyclonedx.parser import BaseParser
+
+
+class PipEnvParser(BaseParser):
+
+    def __init__(self, pipenv_contents: str) -> None:
+        super().__init__()
+
+        pipfile_lock_contents = json.loads(pipenv_contents)
+        pipfile_default: Dict[str, Dict[str, Any]] = pipfile_lock_contents.get('default') or {}
+
+        for (package_name, package_data) in pipfile_default.items():
+            c = Component(
+                name=package_name,
+                version=str(package_data.get('version') or 'unknown').lstrip('='),
+            )
+            if package_data.get('index') == 'pypi' and isinstance(package_data.get('hashes'), list):
+                # Add download location with hashes stored in Pipfile.lock
+                for pip_hash in package_data['hashes']:
+                    ext_ref = ExternalReference(
+                        reference_type=ExternalReferenceType.DISTRIBUTION,
+                        url=c.get_pypi_url(),
+                        comment='Distribution available from pypi.org'
+                    )
+                    ext_ref.add_hash(HashType.from_composite_str(pip_hash))
+                    c.add_external_reference(ext_ref)
+
+            self._components.append(c)
+
+
+class PipEnvFileParser(PipEnvParser):
+
+    def __init__(self, pipenv_lock_filename: str) -> None:
+        with open(pipenv_lock_filename) as r:
+            super(PipEnvFileParser, self).__init__(pipenv_contents=r.read())