From 72f1d5c3fe32f22b32186ab713f190a9a7aa4f7d Mon Sep 17 00:00:00 2001
From: Dorian Turba <froggit.commit.z3jqj@simplelogin.com>
Date: Thu, 7 Sep 2023 10:27:05 +0200
Subject: [PATCH] style depandabot and compare.py
---
pyproject.toml | 9 +
.../depandabot/compare.py | 160 +++++++++++-------
.../depandabot/depandabot.yml | 16 +-
3 files changed, 113 insertions(+), 72 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 8e52531..61f7768 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -22,6 +22,8 @@ dependencies = [
[project.optional-dependencies]
QUALITY = [
+ "black",
+ "mypy",
"pre-commit",
]
@@ -29,3 +31,10 @@ QUALITY = [
strict-config = true
plugins.line-length.enabled = false
+[[tool.mypy.overrides]]
+module = [
+ "packaging.requirements",
+ "packaging.specifiers",
+ "packaging",
+]
+ignore_missing_imports = true
\ No newline at end of file
diff --git a/templates/python/dependancy_management/depandabot/compare.py b/templates/python/dependancy_management/depandabot/compare.py
index 4899f21..22462d2 100644
--- a/templates/python/dependancy_management/depandabot/compare.py
+++ b/templates/python/dependancy_management/depandabot/compare.py
@@ -7,6 +7,11 @@ import argparse
import typing
+class OldNewRequirements(typing.NamedTuple):
+ old: packaging.requirements.Requirement
+ new: packaging.requirements.Requirement
+
+
def clean_lines(f: io.TextIOWrapper) -> typing.Generator[str, None, None]:
for line in f:
# remove comments and hashes
@@ -16,71 +21,16 @@ def clean_lines(f: io.TextIOWrapper) -> typing.Generator[str, None, None]:
yield line.rstrip("\n").rstrip("\\")
-def compare_requirements(
- file: pathlib.Path,
-) -> tuple[
- packaging.requirements.Requirement,
- packaging.requirements.Requirement,
- packaging.requirements.Requirement,
-]:
- set_before = set()
- set_after = set()
- with file.open() as f:
- set_after = {
- packaging.requirements.Requirement(line) for line in clean_lines(f)
- }
-
- set_before = {
- packaging.requirements.Requirement(line)
- for line in clean_lines(retrieve_requirements(file))
- }
-
- d_new = {s.name: s for s in set_after.difference(set_before)}
- d_old = {s.name: s for s in set_before.difference(set_after)}
-
- set_added = {v for k, v in d_new.items() if k not in d_old}
- set_updated = {(d_old[k], v) for k, v in d_new.items() if k in d_old}
- set_removed = {v for k, v in d_old.items() if k not in d_new}
-
- return (set_added, set_updated, set_removed)
-
-
-def generate_changes(
- added: set[packaging.requirements.Requirement],
- updated: set[
- tuple[
- packaging.requirements.Requirement,
- packaging.requirements.Requirement,
- ]
- ],
- removed: set[packaging.requirements.Requirement],
-) -> str:
- description = []
- for a in added:
- description.append(
- f"- Added `{a.name}` version "
- f"`{retrieve_equal_version(a.specifier).version}`"
- )
- for u in updated:
- description.append(
- f"- Bump `{u[0].name}` from "
- f"`{retrieve_equal_version(u[0].specifier).version}` to "
- f"`{retrieve_equal_version(u[1].specifier).version}`"
- )
- for r in removed:
- description.append(
- f"- Removed `{r.name}` version "
- f"`{retrieve_equal_version(r.specifier).version}`"
- )
- return "\r\n".join(description)
-
-
def retrieve_equal_version(
specifier: packaging.specifiers.SpecifierSet,
) -> packaging.specifiers.Specifier:
+ """
+ :raises ValueError: if no equal version is found
+ """
for s in specifier:
if s.operator == "==":
return s
+ raise ValueError("No equal version found")
def retrieve_requirements(file: pathlib.Path) -> io.StringIO:
@@ -92,27 +42,107 @@ def retrieve_requirements(file: pathlib.Path) -> io.StringIO:
return content
+def compare_requirements(
+ requirements: pathlib.Path,
+) -> tuple[
+ set[packaging.requirements.Requirement],
+ set[
+ OldNewRequirements[
+ packaging.requirements.Requirement, packaging.requirements.Requirement
+ ]
+ ],
+ set[packaging.requirements.Requirement],
+]:
+ with requirements.open() as file:
+ set_after = {
+ packaging.requirements.Requirement(line) for line in clean_lines(file)
+ }
+
+ set_before = {
+ packaging.requirements.Requirement(line)
+ for line in clean_lines(retrieve_requirements(requirements))
+ }
+
+ new_requirements = {
+ requirement.name: requirement
+ for requirement in set_after.difference(set_before)
+ }
+ old_requirements = {
+ requirement.name: requirement
+ for requirement in set_before.difference(set_after)
+ }
+
+ added_requirements = {
+ requirement
+ for package_name, requirement in new_requirements.items()
+ if package_name not in old_requirements
+ }
+ updated_requirements = {
+ OldNewRequirements(old_requirements[package_name], requirement)
+ for package_name, requirement in new_requirements.items()
+ if package_name in old_requirements
+ }
+ removed_requirements = {
+ requirement
+ for package_name, requirement in old_requirements.items()
+ if package_name not in new_requirements
+ }
+
+ return added_requirements, updated_requirements, removed_requirements
+
+
def parser() -> argparse.ArgumentParser:
- p = argparse.ArgumentParser()
- p.add_argument(
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
"requirements_file",
nargs="?",
default=pathlib.Path("requirements.txt"),
type=pathlib.Path,
)
- p.add_argument(
+ parser.add_argument(
"-o",
- nargs=1,
+ nargs="?",
default=pathlib.Path("description.md"),
type=pathlib.Path,
required=False,
)
- return p
+ return parser
+
+
+def generate_changes(
+ added_requirements: set[packaging.requirements.Requirement],
+ updated: set[
+ OldNewRequirements[
+ packaging.requirements.Requirement,
+ packaging.requirements.Requirement,
+ ]
+ ],
+ removed: set[packaging.requirements.Requirement],
+) -> str:
+ description = (
+ [
+ f"- Added `{requirement.name}` version "
+ f"`{retrieve_equal_version(requirement.specifier).version}`"
+ for requirement in added_requirements
+ ]
+ + [
+ f"- Bump `{requirements.old.name}` from "
+ f"`{retrieve_equal_version(requirements.old.specifier).version}` to "
+ f"`{retrieve_equal_version(requirements.new.specifier).version}`"
+ for requirements in updated
+ ]
+ + [
+ f"- Removed `{requirement.name}` version "
+ f"`{retrieve_equal_version(requirement.specifier).version}`"
+ for requirement in removed
+ ]
+ )
+ return "\r\n".join(description)
def main():
args = parser().parse_args()
- (added, updated, removed) = compare_requirements(args.requirements_file)
+ added, updated, removed = compare_requirements(args.requirements_file)
args.o.write_text(generate_changes(added, updated, removed))
diff --git a/templates/python/dependancy_management/depandabot/depandabot.yml b/templates/python/dependancy_management/depandabot/depandabot.yml
index 8ef6397..0e8361b 100644
--- a/templates/python/dependancy_management/depandabot/depandabot.yml
+++ b/templates/python/dependancy_management/depandabot/depandabot.yml
@@ -8,17 +8,19 @@ depandabot:
image: python:${IMAGE_TAG}
variables:
- PYTHON_SETUP: pip install pip-tools packaging
+ PYTHON_SETUP: pip install pip-tools
GITLAB_API_URL: "${CI_SERVER_HOST}"
+ REQUIREMENTS_FILE_PATH: "requirements.in"
+ OUTPUT_FILE_PATH: "requirements${IMAGE_TAG}.txt"
script:
- !reference [.python_install, script]
- DEPS_BRANCH="depandabot/requirements-txt/$(date +%s)"
- |
COMMIT_MESSAGE="build(deps): bump new versions"
- - $([ -f ${REQUIREMENTS_FILE_PATH} ]) && ACTION="update" || ACTION="create"
- - pip-compile --quiet -o ${REQUIREMENTS_FILE_PATH}
- - if [ -n "$(git status --porcelain ${REQUIREMENTS_FILE_PATH})" ]; then
+ - $([ -f ${OUTPUT_FILE_PATH} ]) && ACTION="update" || ACTION="create"
+ - pip-compile --quiet -o ${OUTPUT_FILE_PATH} ${REQUIREMENTS_FILE_PATH}
+ - if [ -n "$(git status --porcelain ${OUTPUT_FILE_PATH})" ]; then
- |
curl --header "Authorization: Bearer ${DEPANDABOT_TOKEN}" \
--form "branch=$DEPS_BRANCH" \
@@ -29,10 +31,10 @@ depandabot:
--form "branch=$DEPS_BRANCH" \
--form "commit_message=$COMMIT_MESSAGE" \
--form "actions[][action]=$ACTION" \
- --form "actions[][file_path]=${REQUIREMENTS_FILE_PATH}" \
- --form "actions[][content]=<${REQUIREMENTS_FILE_PATH}" \
+ --form "actions[][file_path]=${OUTPUT_FILE_PATH}" \
+ --form "actions[][content]=<${OUTPUT_FILE_PATH}" \
"https://${GITLAB_API_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/commits"
- - python compare.py -o description.md ${REQUIREMENTS_FILE_PATH}
+ - python compare.py -o description.md ${OUTPUT_FILE_PATH}
- |
curl --header "Authorization: Bearer ${DEPANDABOT_TOKEN}" \
--form "source_branch=$DEPS_BRANCH" \
--
GitLab