diff --git a/templates/python/dependancy_management/depandabot/README.md b/templates/python/dependancy_management/depandabot/README.md index 7e5b71fec83361aaa128952501c56bfe7516caab..63a62aa32b0fbf6e9900a12b7da8713197afb854 100644 --- a/templates/python/dependancy_management/depandabot/README.md +++ b/templates/python/dependancy_management/depandabot/README.md @@ -1,3 +1,5 @@ +# depandabot template + ## Objective The objective of the `depandabot` job is to provide a way to update the requirements.txt file and create a merge request on a Gitlab instance. This reusable job can help speed up other jobs creation and ensure consistent configuration across CI jobs. @@ -34,5 +36,5 @@ requirements311: IMAGE_TAG: "3.11" # override the default IMAGE_TAG variable GITLAB_API_URL: "gitlab.example.com" # override the default GITLAB_API_URL script: - - !reference [script] # reuse the script from the python_install job template + - !reference [script] # reuse the script from the depandabot job template ``` diff --git a/templates/python/dependancy_management/depandabot/compare.py b/templates/python/dependancy_management/depandabot/compare.py new file mode 100644 index 0000000000000000000000000000000000000000..4899f21d2b5bc18236de9f3a09aa185beb183787 --- /dev/null +++ b/templates/python/dependancy_management/depandabot/compare.py @@ -0,0 +1,120 @@ +import packaging.requirements +import packaging.specifiers +import pathlib +import io +import subprocess +import argparse +import typing + + +def clean_lines(f: io.TextIOWrapper) -> typing.Generator[str, None, None]: + for line in f: + # remove comments and hashes + # because packaging.requirements.Requirement can't read hashes + # and comments are useless here + if not line.lstrip().startswith(("#", "--hash")): + yield line.rstrip("\n").rstrip("\\") + + +def compare_requirements( + file: pathlib.Path, +) -> tuple[ + packaging.requirements.Requirement, + packaging.requirements.Requirement, + packaging.requirements.Requirement, +]: + set_before = set() + set_after = set() + with file.open() as f: + set_after = { + packaging.requirements.Requirement(line) for line in clean_lines(f) + } + + set_before = { + packaging.requirements.Requirement(line) + for line in clean_lines(retrieve_requirements(file)) + } + + d_new = {s.name: s for s in set_after.difference(set_before)} + d_old = {s.name: s for s in set_before.difference(set_after)} + + set_added = {v for k, v in d_new.items() if k not in d_old} + set_updated = {(d_old[k], v) for k, v in d_new.items() if k in d_old} + set_removed = {v for k, v in d_old.items() if k not in d_new} + + return (set_added, set_updated, set_removed) + + +def generate_changes( + added: set[packaging.requirements.Requirement], + updated: set[ + tuple[ + packaging.requirements.Requirement, + packaging.requirements.Requirement, + ] + ], + removed: set[packaging.requirements.Requirement], +) -> str: + description = [] + for a in added: + description.append( + f"- Added `{a.name}` version " + f"`{retrieve_equal_version(a.specifier).version}`" + ) + for u in updated: + description.append( + f"- Bump `{u[0].name}` from " + f"`{retrieve_equal_version(u[0].specifier).version}` to " + f"`{retrieve_equal_version(u[1].specifier).version}`" + ) + for r in removed: + description.append( + f"- Removed `{r.name}` version " + f"`{retrieve_equal_version(r.specifier).version}`" + ) + return "\r\n".join(description) + + +def retrieve_equal_version( + specifier: packaging.specifiers.SpecifierSet, +) -> packaging.specifiers.Specifier: + for s in specifier: + if s.operator == "==": + return s + + +def retrieve_requirements(file: pathlib.Path) -> io.StringIO: + p = subprocess.run( + ["git", "show", f"HEAD:{file.name}"], + stdout=subprocess.PIPE, + ) + content = io.StringIO(p.stdout.decode()) + return content + + +def parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser() + p.add_argument( + "requirements_file", + nargs="?", + default=pathlib.Path("requirements.txt"), + type=pathlib.Path, + ) + p.add_argument( + "-o", + nargs=1, + default=pathlib.Path("description.md"), + type=pathlib.Path, + required=False, + ) + return p + + +def main(): + args = parser().parse_args() + (added, updated, removed) = compare_requirements(args.requirements_file) + args.o.write_text(generate_changes(added, updated, removed)) + + +if __name__ == "__main__": + main() diff --git a/templates/python/dependancy_management/depandabot/depandabot.yml b/templates/python/dependancy_management/depandabot/depandabot.yml index 3f7576f3e223d811f8b2d7c6f9a39b8466eb4382..8ef6397b2687c710ca5191326bac186538d1c8a9 100644 --- a/templates/python/dependancy_management/depandabot/depandabot.yml +++ b/templates/python/dependancy_management/depandabot/depandabot.yml @@ -8,7 +8,7 @@ depandabot: image: python:${IMAGE_TAG} variables: - PYTHON_SETUP: pip install pip-tools + PYTHON_SETUP: pip install pip-tools packaging GITLAB_API_URL: "${CI_SERVER_HOST}" script: @@ -32,11 +32,13 @@ depandabot: --form "actions[][file_path]=${REQUIREMENTS_FILE_PATH}" \ --form "actions[][content]=<${REQUIREMENTS_FILE_PATH}" \ "https://${GITLAB_API_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/commits" + - python compare.py -o description.md ${REQUIREMENTS_FILE_PATH} - | curl --header "Authorization: Bearer ${DEPANDABOT_TOKEN}" \ --form "source_branch=$DEPS_BRANCH" \ --form "target_branch=${CI_DEFAULT_BRANCH}" \ --form "title=$COMMIT_MESSAGE" \ + --form "description=<description.md" \ "https://${GITLAB_API_URL}/api/v4/projects/${CI_PROJECT_ID}/merge_requests" - fi