Merge pull request #327 from mostafa/feat/parse-requirements-txt-with…

…-locally-referenced-packages

feat: Change requirements parser

cyclonedx_py/parser/poetry.py

@@ -60,4 +60,3 @@ class PoetryFileParser(PoetryParser):
     def __init__(self, poetry_lock_filename: str) -> None:
         with open(poetry_lock_filename) as r:
             super(PoetryFileParser, self).__init__(poetry_lock_contents=r.read())
-        r.close()

cyclonedx_py/parser/requirements.py

@@ -17,49 +17,64 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright (c) OWASP Foundation. All Rights Reserved.
 
+import os
+import os.path
+from tempfile import NamedTemporaryFile, _TemporaryFileWrapper  # Weak error
+from typing import Any, Optional
+
+from cyclonedx.model import HashType
 from cyclonedx.model.component import Component
 from cyclonedx.parser import BaseParser, ParserWarning
 
 # See https://github.com/package-url/packageurl-python/issues/65
 from packageurl import PackageURL  # type: ignore
-from pkg_resources import parse_requirements as parse_requirements
+from pip_requirements_parser import RequirementsFile  # type: ignore
 
 
 class RequirementsParser(BaseParser):
 
     def __init__(self, requirements_content: str) -> None:
         super().__init__()
+        parsed_rf: Optional[RequirementsFile] = None
+        requirements_file: Optional[_TemporaryFileWrapper[Any]] = None
 
-        requirements = parse_requirements(requirements_content)
-        for requirement in requirements:
-            """
-            @todo
-            Note that the below line will get the first (lowest) version specified in the Requirement and
-            ignore the operator (it might not be ==). This is passed to the Component.
+        if os.path.exists(requirements_content):
+            parsed_rf = RequirementsFile.from_file(
+                requirements_content, include_nested=True)
+        else:
+            requirements_file = NamedTemporaryFile(mode='w+', delete=False)
+            requirements_file.write(requirements_content)
+            requirements_file.close()
 
-            For example if a requirement was listed as: "PickyThing>1.6,<=1.9,!=1.8.6", we'll be interpreting this
-            as if it were written "PickyThing==1.6"
-            """
-            try:
-                (op, version) = requirement.specs[0]
-                self._components.append(Component(
-                    name=requirement.project_name, version=version, purl=PackageURL(
-                        type='pypi', name=requirement.project_name, version=version
-                    )
-                ))
-            except IndexError:
+            parsed_rf = RequirementsFile.from_file(
+                requirements_file.name, include_nested=False)
+
+        for requirement in parsed_rf.requirements:
+            name = requirement.link.url if requirement.is_local_path else requirement.name
+            version = requirement.get_pinned_version or requirement.dumps_specifier()
+            hashes = map(HashType.from_composite_str, requirement.hash_options)
+
+            if not version and not requirement.is_local_path:
                 self._warnings.append(
                     ParserWarning(
-                        item=requirement.project_name,
-                        warning='Requirement \'{}\' does not have a pinned version and cannot be included in your '
-                                'CycloneDX SBOM.'.format(requirement.project_name)
+                        item=name,
+                        warning=(f"Requirement \'{name}\' does not have a pinned "
+                                 "version and cannot be included in your CycloneDX SBOM.")
                     )
                 )
+            else:
+                self._components.append(Component(
+                    name=name,
+                    version=version,
+                    hashes=hashes,
+                    purl=PackageURL(type='pypi', name=name, version=version)
+                ))
+
+        if requirements_file:
+            os.unlink(requirements_file.name)
 
 
 class RequirementsFileParser(RequirementsParser):
 
     def __init__(self, requirements_file: str) -> None:
-        with open(requirements_file) as r:
-            super(RequirementsFileParser, self).__init__(requirements_content=r.read())
-            r.close()
+        super().__init__(requirements_content=requirements_file)

docs/usage.rst

@@ -215,8 +215,8 @@ Requirements
 
 * :py:mod:`cyclonedx_py.parser.requirements.RequirementsParser`: Parses a multiline string that you provide that conforms
   to the ``requirements.txt`` :pep:`508` standard.
-* :py:mod:`cyclonedx_py.parser.requirements.RequirementsFileParser`: Parses a file that you provide the path to that
-  conforms to the ``requirements.txt`` :pep:`508` standard.
+* :py:mod:`cyclonedx_py.parser.requirements.RequirementsFileParser`: Parses a file that you provide the path to that conforms to the ``requirements.txt`` :pep:`508` standard. It supports nested
+files, so if there is a line in your ``requirements.txt`` file with the ``-r requirements-nested.txt`` syntax, it'll parse the nested file as part of the same file.
 
 CycloneDX software bill-of-materials require pinned versions of requirements. If your `requirements.txt` does not have
 pinned versions, warnings will be recorded and the dependencies without pinned versions will be excluded from the

pyproject.toml

@@ -36,6 +36,7 @@ classifiers = [
 [tool.poetry.dependencies]
 python = "^3.6"
 cyclonedx-python-lib = ">= 2.0.0, < 3.0.0"
+pip-requirements-parser = "^31.2.0"
 
 [tool.poetry.dev-dependencies]
 autopep8 = "^1.6.0"

tests/fixtures/requirements-local-and-remote-packages.txt

@@ -0,0 +1,8 @@
+./myproject/certifi # comment
+./myproject/chardet
+-e ./myproject/idna.whl
+./myproject/requests --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804
+./myproject/urllib3
+https://example.com/mypackage.whl
+
+-r requirements-nested.txt

tests/fixtures/requirements-nested.txt

@@ -0,0 +1 @@
+./downloads/numpy-1.9.2-cp34-none-win32.whl

tests/fixtures/requirements-private-packages.txt

@@ -0,0 +1,3 @@
+--extra-index-url https://mypi.org/simple/
+
+mypackage==1.2.3

tests/fixtures/requirements-with-urls.txt

@@ -0,0 +1,4 @@
+git+https://github.com/path/to/package-one@41b95ec#egg=package-one # commit hash
+git+https://github.com/path/to/package-two@master#egg=package-two # master branch
+git+https://github.com/path/to/package-three@0.1#egg=package-three # tag
+git+https://github.com/path/to/package-four@releases/tag/v3.7.1#egg=package-four # release tag

tests/test_parser_requirements.py

@@ -18,66 +18,100 @@
 # Copyright (c) OWASP Foundation. All Rights Reserved.
 
 import os
-import unittest
 from unittest import TestCase
 
-from cyclonedx_py.parser.requirements import RequirementsParser
+from cyclonedx_py.parser.requirements import RequirementsFileParser, RequirementsParser
 
 
 class TestRequirementsParser(TestCase):
 
     def test_simple(self) -> None:
-        with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-simple.txt')) as r:
+        with open(os.path.join(os.path.dirname(__file__),
+                               'fixtures/requirements-simple.txt')) as r:
             parser = RequirementsParser(
                 requirements_content=r.read()
             )
-            r.close()
         self.assertTrue(1, parser.component_count())
         self.assertFalse(parser.has_warnings())
 
     def test_example_1(self) -> None:
-        with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-example-1.txt')) as r:
+        with open(os.path.join(os.path.dirname(__file__),
+                               'fixtures/requirements-example-1.txt')) as r:
             parser = RequirementsParser(
                 requirements_content=r.read()
             )
-            r.close()
         self.assertTrue(3, parser.component_count())
         self.assertFalse(parser.has_warnings())
 
     def test_example_with_comments(self) -> None:
-        with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-with-comments.txt')) as r:
+        with open(os.path.join(os.path.dirname(__file__),
+                               'fixtures/requirements-with-comments.txt')) as r:
             parser = RequirementsParser(
                 requirements_content=r.read()
             )
-            r.close()
         self.assertTrue(5, parser.component_count())
         self.assertFalse(parser.has_warnings())
 
     def test_example_multiline_with_comments(self) -> None:
-        with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-multilines-with-comments.txt')) as r:
+        with open(os.path.join(os.path.dirname(__file__),
+                               'fixtures/requirements-multilines-with-comments.txt')) as r:
             parser = RequirementsParser(
                 requirements_content=r.read()
             )
-            r.close()
         self.assertTrue(5, parser.component_count())
         self.assertFalse(parser.has_warnings())
 
-    @unittest.skip('Not yet supported')
+    def test_example_local_packages(self) -> None:
+        with open(os.path.join(os.path.dirname(__file__),
+                               'fixtures/requirements-local-and-remote-packages.txt')) as r:
+            parser = RequirementsParser(
+                requirements_content=r.read()
+            )
+        self.assertTrue(6, parser.component_count())
+        self.assertFalse(parser.has_warnings())
+
+    def test_example_local_and_nested_packages(self) -> None:
+        # RequirementsFileParser can parse nested requirements files,
+        # but RequirementsParser cannot.
+        parser = RequirementsFileParser(
+            requirements_file='fixtures/requirements-local-and-remote-packages.txt'
+        )
+        self.assertTrue(7, parser.component_count())
+        self.assertFalse(parser.has_warnings())
+
+    def test_example_private_packages(self) -> None:
+        with open(os.path.join(os.path.dirname(__file__),
+                               'fixtures/requirements-private-packages.txt')) as r:
+            parser = RequirementsParser(
+                requirements_content=r.read()
+            )
+        self.assertTrue(1, parser.component_count())
+        self.assertFalse(parser.has_warnings())
+
+    def test_example_with_urls(self) -> None:
+        with open(os.path.join(os.path.dirname(__file__),
+                               'fixtures/requirements-with-urls.txt')) as r:
+            parser = RequirementsParser(
+                requirements_content=r.read()
+            )
+        self.assertTrue(4, parser.component_count())
+        self.assertFalse(parser.has_warnings())
+
     def test_example_with_hashes(self) -> None:
-        with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-with-hashes.txt')) as r:
+        with open(os.path.join(os.path.dirname(__file__),
+                               'fixtures/requirements-with-hashes.txt')) as r:
             parser = RequirementsParser(
                 requirements_content=r.read()
             )
-            r.close()
         self.assertTrue(5, parser.component_count())
         self.assertFalse(parser.has_warnings())
 
     def test_example_without_pinned_versions(self) -> None:
-        with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-without-pinned-versions.txt')) as r:
+        with open(os.path.join(os.path.dirname(__file__),
+                               'fixtures/requirements-without-pinned-versions.txt')) as r:
             parser = RequirementsParser(
                 requirements_content=r.read()
             )
-            r.close()
         self.assertTrue(2, parser.component_count())
         self.assertTrue(parser.has_warnings())
         self.assertEqual(3, len(parser.get_warnings()))