Automatically use requirements.in if the project uses a requirements.txt + requirements.in-setup (#641)

fpgmaas · web-flow · commit 2ac789fe8ac0 · 2024-04-04T14:10:49.000+02:00
diff --git a/docs/usage.md b/docs/usage.md
@@ -31,9 +31,9 @@ To determine the project's dependencies, _deptry_ will scan the directory it is
 3. If a `pyproject.toml` file with a `[project]` section is found, _deptry_ will assume it uses [PEP 621](https://peps.python.org/pep-0621/) for dependency specification and extract:
     - dependencies from `[project.dependencies]` and `[project.optional-dependencies]`.
     - development dependencies from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument.
-4. If a `requirements.txt` file is found, _deptry_ will extract:
-    - dependencies from it
-    - development dependencies from `dev-dependencies.txt` and `dependencies-dev.txt`, if any exist
+4. If a `requirements.in` or `requirements.txt` file is found, _deptry_ will:
+    - extract dependencies from that file.
+    - extract development dependencies from `dev-dependencies.txt` and `dependencies-dev.txt`, if any exist
 
 _deptry_ can be configured to look for `pip` requirements files with other names or in other directories.
 See [Requirements files](#requirements-files) and [Requirements files dev](#requirements-files-dev).
diff --git a/pdm.lock b/pdm.lock
diff --git a/python/deptry/cli.py b/python/deptry/cli.py
@@ -26,6 +26,8 @@
 
 DEFAULT_EXCLUDE = ("venv", r"\.venv", r"\.direnv", "tests", r"\.git", r"setup\.py")
 
+DEFAULT_REQUIREMENTS_FILES = ("requirements.txt",)
+
 
 class CommaSeparatedTupleParamType(click.ParamType):
     """
@@ -159,8 +161,7 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b
     help=f"""A regular expression for directories or files in which .py files should not be scanned for imports to determine if there are dependency issues.
     Can be used multiple times by specifying the argument multiple times. re.match() is used to match the expressions, which by default checks for a match only at the beginning of a string.
     For example: `deptry . -e ".*/foo/" -e bar"` Note that this overwrites the defaults.
-    [default: {", ".join(DEFAULT_EXCLUDE)}
-    """,
+    [default: {", ".join(DEFAULT_EXCLUDE)}]""",
 )
 @click.option(
     "--extend-exclude",
@@ -196,10 +197,9 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b
     "--requirements-files",
     "-rf",
     type=COMMA_SEPARATED_TUPLE,
-    help=""".txt files to scan for dependencies. If a file called pyproject.toml with a [tool.poetry.dependencies] or [project] section is found, this argument is ignored
-    and the dependencies are extracted from the pyproject.toml file instead. Can be multiple e.g. `deptry . --requirements-txt req/prod.txt,req/extra.txt`""",
-    default=("requirements.txt",),
-    show_default=True,
+    help=f""".txt files to scan for dependencies. If a file called pyproject.toml with a [tool.poetry.dependencies] or [project] section is found, this argument is ignored
+    and the dependencies are extracted from the pyproject.toml file instead. Can be multiple e.g. `deptry . --requirements-txt req/prod.txt,req/extra.txt`
+    [default: {", ".join(DEFAULT_REQUIREMENTS_FILES)}]""",
 )
 @click.option(
     "--requirements-files-dev",
@@ -292,7 +292,8 @@ def deptry(
         ignore_notebooks=ignore_notebooks,
         ignore=ignore,
         per_rule_ignores=per_rule_ignores,
-        requirements_files=requirements_txt or requirements_files,
+        requirements_files=(requirements_txt or requirements_files) or DEFAULT_REQUIREMENTS_FILES,
+        using_default_requirements_files=not (requirements_txt or requirements_files),
         requirements_files_dev=requirements_txt_dev or requirements_files_dev,
         known_first_party=known_first_party,
         json_output=json_output,
diff --git a/python/deptry/core.py b/python/deptry/core.py
@@ -34,6 +34,7 @@ class Core:
     using_default_exclude: bool
     ignore_notebooks: bool
     requirements_files: tuple[str, ...]
+    using_default_requirements_files: bool
     requirements_files_dev: tuple[str, ...]
     known_first_party: tuple[str, ...]
     json_output: str
@@ -48,6 +49,7 @@ def run(self) -> None:
             self.package_module_name_map,
             self.pep621_dev_dependency_groups,
             self.requirements_files,
+            self.using_default_requirements_files,
             self.requirements_files_dev,
         ).build()
 
diff --git a/python/deptry/dependency_getter/builder.py b/python/deptry/dependency_getter/builder.py
@@ -32,6 +32,7 @@ class DependencyGetterBuilder:
     package_module_name_map: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
     pep621_dev_dependency_groups: tuple[str, ...] = ()
     requirements_files: tuple[str, ...] = ()
+    using_default_requirements_files: bool = True
     requirements_files_dev: tuple[str, ...] = ()
 
     def build(self) -> DependencyGetter:
@@ -51,9 +52,10 @@ def build(self) -> DependencyGetter:
                     self.config, self.package_module_name_map, self.pep621_dev_dependency_groups
                 )
 
-        if self._project_uses_requirements_files():
+        check, requirements_files = self._project_uses_requirements_files()
+        if check:
             return RequirementsTxtDependencyGetter(
-                self.config, self.package_module_name_map, self.requirements_files, self.requirements_files_dev
+                self.config, self.package_module_name_map, requirements_files, self.requirements_files_dev
             )
 
         raise DependencySpecificationNotFoundError(self.requirements_files)
@@ -114,11 +116,26 @@ def _project_uses_pep_621(pyproject_toml: dict[str, Any]) -> bool:
         )
         return False
 
-    def _project_uses_requirements_files(self) -> bool:
+    def _project_uses_requirements_files(self) -> tuple[bool, tuple[str, ...]]:
+        """
+        Tools like `pip-tools` and `uv` work with a setup in which a `requirements.in` is compiled into a `requirements.txt`, which then
+        contains pinned versions for all transitive dependencies. If the user did not explicitly specify the argument `requirements-files`,
+        but there is a `requirements.in` present, it is highly likely that the user wants to use the `requirements.in` file so we set
+        `requirements-files` to that instead.
+        """
+        if self.using_default_requirements_files and Path("requirements.in").is_file():
+            logging.info(
+                "Detected a 'requirements.in' file in the project and no 'requirements-files' were explicitly specified. "
+                "Automatically using 'requirements.in' as the source for the project's dependencies. To specify a different source for "
+                "the project's dependencies, use the '--requirements-files' option."
+            )
+            return True, ("requirements.in",)
+
         check = any(Path(requirements_files).is_file() for requirements_files in self.requirements_files)
         if check:
             logging.debug(
                 "Dependency specification found in '%s'. Will use this to determine the project's dependencies.\n",
                 ", ".join(self.requirements_files),
             )
-        return check
+            return True, self.requirements_files
+        return False, ()
diff --git a/tests/data/project_with_requirements_in/pyproject.toml b/tests/data/project_with_requirements_in/pyproject.toml
@@ -0,0 +1,5 @@
+[tool.black]
+line-length = 120
+
+[tool.deptry.per_rule_ignores]
+DEP001 = ["toml"]
diff --git a/tests/data/project_with_requirements_in/requirements-dev.txt b/tests/data/project_with_requirements_in/requirements-dev.txt
@@ -0,0 +1 @@
+black==22.6.0
diff --git a/tests/data/project_with_requirements_in/requirements.in b/tests/data/project_with_requirements_in/requirements.in
@@ -0,0 +1,4 @@
+click==8.1.3
+isort==5.10.1
+urllib3
+pandas
diff --git a/tests/data/project_with_requirements_in/requirements.txt b/tests/data/project_with_requirements_in/requirements.txt
@@ -0,0 +1,7 @@
+# Generated from requirements.in
+click==8.1.3
+isort==5.10.1
+urllib3
+requests
+pandas
+numpy
diff --git a/tests/data/project_with_requirements_in/src/main.py b/tests/data/project_with_requirements_in/src/main.py
@@ -0,0 +1,8 @@
+from os import chdir, walk
+from pathlib import Path
+
+import black
+import click
+import white as w
+from urllib3 import contrib
+import requests
diff --git a/tests/data/project_with_requirements_in/src/notebook.ipynb b/tests/data/project_with_requirements_in/src/notebook.ipynb
@@ -0,0 +1,37 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "9f4924ec-2200-4801-9d49-d4833651cbc4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import click\n",
+    "from urllib3 import contrib\n",
+    "import toml"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/tests/functional/cli/test_cli_requirements_in.py b/tests/functional/cli/test_cli_requirements_in.py
@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+
+from tests.functional.utils import Project
+from tests.utils import get_issues_report
+
+if TYPE_CHECKING:
+    from tests.utils import PipVenvFactory
+
+
+@pytest.mark.xdist_group(name=Project.REQUIREMENTS_IN)
+def test_cli_single_requirements_files(pip_venv_factory: PipVenvFactory) -> None:
+    """
+    in this case, deptry should recognize that there is a `requirements.in` in the project, and
+    use that as the source of the dependencies.
+    """
+    with pip_venv_factory(
+        Project.REQUIREMENTS_IN,
+        install_command=("pip install -r requirements.txt -r requirements-dev.txt"),
+    ) as virtual_env:
+        issue_report = f"{uuid.uuid4()}.json"
+        result = virtual_env.run(f"deptry . -o {issue_report}")
+
+        assert result.returncode == 1
+        assert get_issues_report(Path(issue_report)) == [
+            {
+                "error": {"code": "DEP002", "message": "'isort' defined as a dependency but not used in the codebase"},
+                "module": "isort",
+                "location": {"file": str(Path("requirements.in")), "line": None, "column": None},
+            },
+            {
+                "error": {"code": "DEP002", "message": "'pandas' defined as a dependency but not used in the codebase"},
+                "module": "pandas",
+                "location": {"file": str(Path("requirements.in")), "line": None, "column": None},
+            },
+            {
+                "error": {"code": "DEP004", "message": "'black' imported but declared as a dev dependency"},
+                "module": "black",
+                "location": {"file": str(Path("src/main.py")), "line": 4, "column": 8},
+            },
+            {
+                "error": {"code": "DEP001", "message": "'white' imported but missing from the dependency definitions"},
+                "module": "white",
+                "location": {"file": str(Path("src/main.py")), "line": 6, "column": 8},
+            },
+            {
+                "error": {"code": "DEP003", "message": "'requests' imported but it is a transitive dependency"},
+                "module": "requests",
+                "location": {"file": str(Path("src/main.py")), "line": 8, "column": 8},
+            },
+        ]
+
+
+@pytest.mark.xdist_group(name=Project.REQUIREMENTS_IN)
+def test_cli_multiple_requirements_files(pip_venv_factory: PipVenvFactory) -> None:
+    """
+    in this case, deptry recognizes that there is a `requirements.in` in the project, but the user
+    can overwrite that with '--requirements-files requirements.txt', so it still takes requirements.txt as the source
+    for the project's dependencies.
+    """
+    with pip_venv_factory(
+        Project.REQUIREMENTS_IN,
+        install_command=("pip install -r requirements.txt -r requirements-dev.txt"),
+    ) as virtual_env:
+        issue_report = f"{uuid.uuid4()}.json"
+        result = virtual_env.run(f"deptry . --requirements-files requirements.txt -o {issue_report}")
+
+        assert result.returncode == 1
+        assert get_issues_report(Path(issue_report)) == [
+            {
+                "error": {"code": "DEP002", "message": "'isort' defined as a dependency but not used in the codebase"},
+                "module": "isort",
+                "location": {"file": str(Path("requirements.txt")), "line": None, "column": None},
+            },
+            {
+                "error": {"code": "DEP002", "message": "'pandas' defined as a dependency but not used in the codebase"},
+                "module": "pandas",
+                "location": {"file": str(Path("requirements.txt")), "line": None, "column": None},
+            },
+            {
+                "error": {"code": "DEP002", "message": "'numpy' defined as a dependency but not used in the codebase"},
+                "module": "numpy",
+                "location": {"file": str(Path("requirements.txt")), "line": None, "column": None},
+            },
+            {
+                "error": {"code": "DEP004", "message": "'black' imported but declared as a dev dependency"},
+                "module": "black",
+                "location": {"file": str(Path("src/main.py")), "line": 4, "column": 8},
+            },
+            {
+                "error": {"code": "DEP001", "message": "'white' imported but missing from the dependency definitions"},
+                "module": "white",
+                "location": {"file": str(Path("src/main.py")), "line": 6, "column": 8},
+            },
+        ]
diff --git a/tests/functional/utils.py b/tests/functional/utils.py
@@ -15,6 +15,7 @@ class Project(str, Enum):
     POETRY = "project_with_poetry"
     PYPROJECT_DIFFERENT_DIRECTORY = "project_with_pyproject_different_directory"
     REQUIREMENTS_TXT = "project_with_requirements_txt"
+    REQUIREMENTS_IN = "project_with_requirements_in"
     SRC_DIRECTORY = "project_with_src_directory"
 
     def __str__(self) -> str:
diff --git a/tests/unit/dependency_getter/test_builder.py b/tests/unit/dependency_getter/test_builder.py
@@ -3,6 +3,7 @@
 import logging
 import re
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import pytest
 
@@ -14,6 +15,9 @@
 from deptry.exceptions import DependencySpecificationNotFoundError
 from tests.utils import run_within_dir
 
+if TYPE_CHECKING:
+    from _pytest.logging import LogCaptureFixture
+
 
 def test_poetry(tmp_path: Path) -> None:
     with run_within_dir(tmp_path):
@@ -131,3 +135,31 @@ def test_dependency_specification_not_found_raises_exception(tmp_path: Path, cap
             " dependencies."
         ),
     ]
+
+
+def test_check_for_requirements_in_file_with_requirements_in(tmp_path: Path, caplog: LogCaptureFixture) -> None:
+    with run_within_dir(tmp_path):
+        # Setup: Create a requirements.in file in the temporary directory
+        requirements_in_path = Path("requirements.in")
+        requirements_in_path.touch()
+        requirements_txt_path = Path("requirements.txt")
+        requirements_txt_path.touch()
+
+        # Use caplog to capture logging at the INFO level
+        with caplog.at_level(logging.INFO):
+            spec = DependencyGetterBuilder(
+                config=Path("pyproject.toml"),
+                requirements_files=("requirements.txt",),
+                using_default_requirements_files=True,
+            ).build()
+
+        # Assert that requirements_files is updated correctly
+        assert spec.requirements_files == ("requirements.in",)  # type: ignore[attr-defined]
+
+        # Assert that the expected log message is present
+        expected_log = (
+            "Detected a 'requirements.in' file in the project and no 'requirements-files' were explicitly specified. "
+            "Automatically using 'requirements.in' as the source for the project's dependencies. To specify a different source for "
+            "the project's dependencies, use the '--requirements-files' option."
+        )
+        assert expected_log in caplog.text
diff --git a/tests/unit/deprecate/test_requirements_txt.py b/tests/unit/deprecate/test_requirements_txt.py
@@ -24,6 +24,7 @@
     "json_output": ANY,
     "package_module_name_map": ANY,
     "pep621_dev_dependency_groups": ANY,
+    "using_default_requirements_files": ANY,
 }
 
 
diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py
@@ -74,6 +74,7 @@ def test__get_local_modules(
                 json_output="",
                 package_module_name_map={},
                 pep621_dev_dependency_groups=(),
+                using_default_requirements_files=True,
             )._get_local_modules()
             == expected
         )

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +click==8.1.3
 +isort==5.10.1
 +urllib3
 +pandas