diff --git a/.ci/ansible/filter/repr.py b/.ci/ansible/filter/repr.py index 8455c3442..c8c1678dd 100644 --- a/.ci/ansible/filter/repr.py +++ b/.ci/ansible/filter/repr.py @@ -1,4 +1,5 @@ from __future__ import absolute_import, division, print_function + from packaging.version import parse as parse_version __metaclass__ = type diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 66c494e97..83e197aa7 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -7,11 +7,12 @@ import argparse import fileinput -import urllib.request import sys +import urllib.request + +import yaml from packaging.requirements import Requirement from packaging.version import Version -import yaml try: import tomllib diff --git a/.ci/scripts/check_gettext.sh b/.ci/scripts/check_gettext.sh deleted file mode 100755 index 39bdeb04f..000000000 --- a/.ci/scripts/check_gettext.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../.. - -set -uv - -MATCHES=$(grep -n -r --include \*.py "_(f") - -if [ $? -ne 1 ]; then - printf "\nERROR: Detected mix of f-strings and gettext:\n" - echo "$MATCHES" - exit 1 -fi diff --git a/.ci/scripts/check_pulpcore_imports.sh b/.ci/scripts/check_pulpcore_imports.sh index 5d9c6f481..e1867c625 100755 --- a/.ci/scripts/check_pulpcore_imports.sh +++ b/.ci/scripts/check_pulpcore_imports.sh @@ -10,10 +10,10 @@ # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../.. -set -uv +set -u # check for imports not from pulpcore.plugin. exclude tests -MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin") +MATCHES="$(grep -n -r --include \*.py "from pulpcore.*import" pulp_python | grep -v "tests\|plugin")" if [ $? -ne 1 ]; then printf "\nERROR: Detected bad imports from pulpcore:\n" diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 6e0799952..0825480e2 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -9,16 +9,16 @@ # /// import argparse -import re import os +import re import sys import tomllib import typing as t from pathlib import Path import yaml -from packaging.version import Version from git import Repo +from packaging.version import Version RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" Y_CHANGELOG_EXTS = [".feature"] @@ -157,9 +157,9 @@ def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int: if reasons: curr_version = Version(last_tag) - assert curr_version.base_version.startswith( - branch - ), "Current-version has to belong to the current branch!" + assert curr_version.base_version.startswith(branch), ( + "Current-version has to belong to the current branch!" + ) next_version = Version(f"{branch}.{curr_version.micro + 1}") print( f"A Z-release is needed for {branch}, " diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index eca496879..b497b4e23 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -7,12 +7,12 @@ import tomllib import warnings + from packaging.requirements import Requirement CHECK_MATRIX = [ ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), - ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), ("doc_requirements.txt", False, True, False), ("lint_requirements.txt", False, True, True), diff --git a/.ci/scripts/schema.py b/.ci/scripts/schema.py index 91191c5ed..9c8e11b2e 100644 --- a/.ci/scripts/schema.py +++ b/.ci/scripts/schema.py @@ -9,6 +9,7 @@ """ import json + from drf_spectacular.validation import JSON_SCHEMA_SPEC_PATH with open(JSON_SCHEMA_SPEC_PATH) as fh: diff --git a/.ci/scripts/skip_tests.py b/.ci/scripts/skip_tests.py index a68d000d6..380a3da9f 100755 --- a/.ci/scripts/skip_tests.py +++ b/.ci/scripts/skip_tests.py @@ -15,12 +15,13 @@ *: Error """ -import sys +import argparse import os import re -import git +import sys import textwrap -import argparse + +import git DOC_PATTERNS = [ r"^docs/", diff --git a/.ci/scripts/update_github.py b/.ci/scripts/update_github.py index b298ad836..f5e81a5fb 100755 --- a/.ci/scripts/update_github.py +++ b/.ci/scripts/update_github.py @@ -6,6 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template import os + from github import Github g = Github(os.environ.get("GITHUB_TOKEN")) diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index a4dc9004a..2aab90855 100644 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -6,9 +6,9 @@ import subprocess import sys import tomllib -import yaml from pathlib import Path +import yaml from github import Github diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 9e00b3faa..000000000 --- a/.flake8 +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template -[flake8] -exclude = ./docs/*,*/migrations/* -per-file-ignores = */__init__.py: F401 - -ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402,F824 -max-line-length = 100 - -# Flake8 builtin codes -# -------------------- -# E203: no whitespace around ':'. disabled until https://github.com/PyCQA/pycodestyle/issues/373 is fixed -# W503: This enforces operators before line breaks which is not pep8 or black compatible. -# F824: 'nonlocal' is unused: name is never assigned in scope - -# Flake8-quotes extension codes -# ----------------------------- -# Q000: double or single quotes only, default is double (don't want to enforce this) -# Q003: Change outer quotes to avoid escaping inner quotes - -# Flake8-docstring extension codes -# -------------------------------- -# D100: missing docstring in public module -# D104: missing docstring in public package -# D106: missing docstring in public nested class (complains about "class Meta:" and documenting those is silly) -# D200: one-line docstring should fit on one line with quotes -# D205: 1 blank line required between summary line and description -# D400: First line should end with a period -# D401: first line should be imperative (nitpicky) -# D402: first line should not be the function’s “signature” (false positives) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index bd1f718c1..8da63002e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -36,20 +36,13 @@ jobs: run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows - # run black separately from flake8 to get a diff - - name: "Run black" + - name: "Check formating" run: | - black --version - black --check --diff . + ruff format --check --diff - # Lint code. - - name: "Run flake8" + - name: "Lint code" run: | - flake8 - - - name: "Check for common gettext problems" - run: | - sh .ci/scripts/check_gettext.sh + ruff check - name: "Run extra lint checks" run: | diff --git a/.github/workflows/scripts/stage-changelog-for-default-branch.py b/.github/workflows/scripts/stage-changelog-for-default-branch.py index 1a21419d4..255c58424 100755 --- a/.github/workflows/scripts/stage-changelog-for-default-branch.py +++ b/.github/workflows/scripts/stage-changelog-for-default-branch.py @@ -12,13 +12,15 @@ from git import Repo from git.exc import GitCommandError -helper = textwrap.dedent("""\ +helper = textwrap.dedent( + """\ Stage the changelog for a release on main branch. Example: $ python .github/workflows/scripts/stage-changelog-for-default-branch.py 3.4.0 - """) + """ +) parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) diff --git a/.github/workflows/scripts/update_backport_labels.py b/.github/workflows/scripts/update_backport_labels.py index 967984a42..063314b6c 100755 --- a/.github/workflows/scripts/update_backport_labels.py +++ b/.github/workflows/scripts/update_backport_labels.py @@ -5,10 +5,11 @@ # # For more info visit https://github.com/pulp/plugin_template +import os +import random + import requests import yaml -import random -import os def random_color(): diff --git a/dev_requirements.txt b/dev_requirements.txt deleted file mode 100644 index 0d7d2e745..000000000 --- a/dev_requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -black -check-manifest -flake8 -flake8-docstrings -flake8-tuple -flake8-quotes -# pin pydocstyle until https://gitlab.com/pycqa/flake8-docstrings/issues/36 is resolved -pydocstyle<4 -requests diff --git a/lint_requirements.txt b/lint_requirements.txt index 5c38ff60f..fd592d7d8 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,10 +5,8 @@ # # For more info visit https://github.com/pulp/plugin_template -black~=26.3 # Pin style to the year. https://black.readthedocs.io/en/stable/faq.html#how-stable-is-black-s-style bump-my-version check-manifest -flake8 -flake8-black packaging +ruff yamllint diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index b04270560..fdcf1913b 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -1,6 +1,8 @@ +from gettext import gettext as _ + from django.db.models.signals import post_migrate + from pulpcore.plugin import PulpPluginAppConfig -from gettext import gettext as _ class PulpPythonPluginAppConfig(PulpPluginAppConfig): @@ -26,7 +28,7 @@ def ready(self): # TODO: Remove this when https://github.com/pulp/pulpcore/issues/5500 is resolved def _populate_pypi_access_policies(sender, apps, verbosity, **kwargs): - from pulp_python.app.pypi.views import PyPIView, SimpleView, UploadView, MetadataView + from pulp_python.app.pypi.views import MetadataView, PyPIView, SimpleView, UploadView try: AccessPolicy = apps.get_model("core", "AccessPolicy") diff --git a/pulp_python/app/management/commands/repair-python-metadata.py b/pulp_python/app/management/commands/repair-python-metadata.py index 0439d2da2..ed4dec85c 100644 --- a/pulp_python/app/management/commands/repair-python-metadata.py +++ b/pulp_python/app/management/commands/repair-python-metadata.py @@ -1,11 +1,12 @@ -import re import os -from django.core.management import BaseCommand, CommandError +import re from gettext import gettext as _ from django.conf import settings +from django.core.management import BaseCommand, CommandError from pulpcore.plugin.util import extract_pk + from pulp_python.app.models import PythonPackageContent, PythonRepository from pulp_python.app.utils import artifact_to_python_content_data diff --git a/pulp_python/app/modelresource.py b/pulp_python/app/modelresource.py index 6533c8810..bc5f41781 100644 --- a/pulp_python/app/modelresource.py +++ b/pulp_python/app/modelresource.py @@ -1,6 +1,7 @@ from pulpcore.plugin.importexport import BaseContentResource from pulpcore.plugin.modelresources import RepositoryResource from pulpcore.plugin.util import get_domain + from pulp_python.app.models import ( PythonPackageContent, PythonRepository, diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index fefa655bf..fa45835db 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -1,44 +1,45 @@ import hashlib import json from logging import getLogger +from pathlib import PurePath from aiohttp.web import json_response +from django.conf import settings from django.contrib.postgres.fields import ArrayField from django.core.exceptions import ObjectDoesNotExist from django.db import models -from django.conf import settings from django_lifecycle import ( BEFORE_SAVE, hook, ) from rest_framework.serializers import ValidationError + from pulpcore.plugin.models import ( AutoAddObjPermsMixin, BaseModel, Content, - Publication, Distribution, + Publication, Remote, Repository, ) +from pulpcore.plugin.repo_version_utils import ( + collect_duplicates, + remove_duplicates, + validate_repo_version, +) from pulpcore.plugin.responses import ArtifactResponse +from pulpcore.plugin.util import get_domain, get_domain_pk -from pathlib import PurePath from .provenance import Provenance from .utils import ( - artifact_to_python_content_data, + PYPI_LAST_SERIAL, + PYPI_SERIAL_CONSTANT, artifact_to_metadata_artifact, + artifact_to_python_content_data, canonicalize_name, python_content_to_json, - PYPI_LAST_SERIAL, - PYPI_SERIAL_CONSTANT, -) -from pulpcore.plugin.repo_version_utils import ( - collect_duplicates, - remove_duplicates, - validate_repo_version, ) -from pulpcore.plugin.util import get_domain_pk, get_domain log = getLogger(__name__) @@ -453,8 +454,9 @@ def check_blocklist_for_packages(self, packages): break if blocked: raise ValidationError( - "Blocklisted packages cannot be added to this repository: " - "{}".format(", ".join(blocked)) + "Blocklisted packages cannot be added to this repository: {}".format( + ", ".join(blocked) + ) ) diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py index 5c910d5b1..746606a9d 100644 --- a/pulp_python/app/pypi/serializers.py +++ b/pulp_python/app/pypi/serializers.py @@ -1,13 +1,15 @@ import logging from gettext import gettext as _ -from rest_framework import serializers +from django.db.utils import IntegrityError from pydantic import TypeAdapter, ValidationError -from pulp_python.app.provenance import Attestation -from pulp_python.app.utils import DIST_EXTENSIONS, SUPPORTED_METADATA_VERSIONS +from rest_framework import serializers + from pulpcore.plugin.models import Artifact from pulpcore.plugin.util import get_domain -from django.db.utils import IntegrityError + +from pulp_python.app.provenance import Attestation +from pulp_python.app.utils import DIST_EXTENSIONS, SUPPORTED_METADATA_VERSIONS log = logging.getLogger(__name__) @@ -110,7 +112,7 @@ def validate(self, data): attestations = TypeAdapter(list[Attestation]).validate_python(attestations) except ValidationError as e: raise serializers.ValidationError( - {"attestations": _("The uploaded attestations are not valid: {}".format(e))} + {"attestations": _("The uploaded attestations are not valid: {}").format(e)} ) sha256 = data.get("sha256_digest") diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index 87f03dac5..04bcc1653 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -1,60 +1,59 @@ import logging - -from rest_framework.viewsets import ViewSet -from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer, TemplateHTMLRenderer -from rest_framework.response import Response -from rest_framework.exceptions import NotAcceptable -from django.core.exceptions import ObjectDoesNotExist -from django.shortcuts import redirect -from datetime import datetime, timezone, timedelta +from datetime import datetime, timedelta, timezone +from itertools import chain +from pathlib import PurePath +from urllib.parse import urljoin, urlparse, urlunsplit from django.contrib.sessions.models import Session +from django.core.exceptions import ObjectDoesNotExist from django.db import transaction from django.db.utils import DatabaseError from django.http.response import ( Http404, - HttpResponseNotFound, - HttpResponseForbidden, + HttpResponse, HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseNotFound, StreamingHttpResponse, - HttpResponse, ) +from django.shortcuts import redirect from drf_spectacular.utils import extend_schema from dynaconf import settings -from itertools import chain from packaging.utils import canonicalize_name -from urllib.parse import urljoin, urlparse, urlunsplit -from pathlib import PurePath +from rest_framework.exceptions import NotAcceptable +from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer, TemplateHTMLRenderer +from rest_framework.response import Response +from rest_framework.viewsets import ViewSet -from pulpcore.plugin.viewsets import OperationPostponedResponse from pulpcore.plugin.tasking import dispatch from pulpcore.plugin.util import get_domain, get_url +from pulpcore.plugin.viewsets import OperationPostponedResponse + +from pulp_python.app import tasks from pulp_python.app.models import ( + PackageProvenance, PythonDistribution, PythonPackageContent, PythonPublication, - PackageProvenance, ) from pulp_python.app.pypi.serializers import ( - SummarySerializer, PackageMetadataSerializer, PackageUploadSerializer, PackageUploadTaskSerializer, + SummarySerializer, ) from pulp_python.app.utils import ( - write_simple_index, - write_simple_index_json, - write_simple_detail, - write_simple_detail_json, - python_content_to_json, PYPI_LAST_SERIAL, PYPI_SERIAL_CONSTANT, get_remote_package_filter, get_remote_simple_page, + python_content_to_json, + write_simple_detail, + write_simple_detail_json, + write_simple_index, + write_simple_index_json, ) -from pulp_python.app import tasks - log = logging.getLogger(__name__) ORIGIN_HOST = settings.CONTENT_ORIGIN if settings.CONTENT_ORIGIN else settings.PYPI_API_HOSTNAME diff --git a/pulp_python/app/replica.py b/pulp_python/app/replica.py index da63c1810..1ddcc725c 100644 --- a/pulp_python/app/replica.py +++ b/pulp_python/app/replica.py @@ -1,10 +1,11 @@ -from pulpcore.plugin.replica import Replicator - from pulp_glue.python.context import ( PulpPythonDistributionContext, PulpPythonPublicationContext, PulpPythonRepositoryContext, ) + +from pulpcore.plugin.replica import Replicator + from pulp_python.app.models import PythonDistribution, PythonRemote, PythonRepository from pulp_python.app.tasks import sync as python_sync diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 35fd5ca5b..14784db39 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -2,33 +2,34 @@ import os import tempfile from gettext import gettext as _ +from urllib.parse import urljoin + from django.conf import settings from django.db.utils import IntegrityError from drf_spectacular.utils import extend_schema_serializer from packaging.requirements import Requirement -from packaging.version import Version, InvalidVersion -from rest_framework import serializers -from pypi_attestations import AttestationError +from packaging.version import InvalidVersion, Version from pydantic import TypeAdapter, ValidationError -from urllib.parse import urljoin +from pypi_attestations import AttestationError +from rest_framework import serializers from pulpcore.plugin import models as core_models from pulpcore.plugin import serializers as core_serializers -from pulpcore.plugin.util import get_domain, get_prn, get_current_authenticated_user, reverse +from pulpcore.plugin.util import get_current_authenticated_user, get_domain, get_prn, reverse from pulp_python.app import models as python_models -from pulp_python.app.utils import canonicalize_name from pulp_python.app.provenance import ( + AnyPublisher, Attestation, + AttestationBundle, Provenance, verify_provenance, - AttestationBundle, - AnyPublisher, ) from pulp_python.app.utils import ( DIST_EXTENSIONS, artifact_to_metadata_artifact, artifact_to_python_content_data, + canonicalize_name, get_project_metadata_from_file, parse_project_metadata, ) @@ -238,7 +239,7 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_( - "The maintainer's name at a minimum; " "additional contact information may be provided." + "The maintainer's name at a minimum; additional contact information may be provided." ), ) maintainer_email = serializers.CharField( @@ -387,7 +388,7 @@ def validate_attestations(self, value): else: attestations = TypeAdapter(list[Attestation]).validate_python(value) except ValidationError as e: - raise serializers.ValidationError(_("Invalid attestations: {}".format(e))) + raise serializers.ValidationError(_("Invalid attestations: {}").format(e)) return attestations def handle_attestations(self, filename, sha256, attestations, offline=True): @@ -400,7 +401,7 @@ def handle_attestations(self, filename, sha256, attestations, offline=True): verify_provenance(filename, sha256, provenance, offline=offline) except AttestationError as e: raise serializers.ValidationError( - {"attestations": _("Attestations failed verification: {}".format(e))} + {"attestations": _("Attestations failed verification: {}").format(e)} ) return provenance.model_dump(mode="json") @@ -655,13 +656,13 @@ def deferred_validate(self, data): data["provenance"] = provenance.model_dump(mode="json") except ValidationError as e: raise serializers.ValidationError( - _("The uploaded provenance is not valid: {}".format(e)) + _("The uploaded provenance is not valid: {}").format(e) ) if data.pop("verify"): try: verify_provenance(data["package"].filename, data["package"].sha256, provenance) except AttestationError as e: - raise serializers.ValidationError(_("Provenance verification failed: {}".format(e))) + raise serializers.ValidationError(_("Provenance verification failed: {}").format(e)) return data def retrieve(self, validated_data): @@ -729,7 +730,7 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer): package_types = MultipleChoiceArrayField( required=False, help_text=_( - "The package types to sync for Python content. Leave blank to get every" "package type." + "The package types to sync for Python content. Leave blank to get everypackage type." ), choices=python_models.PACKAGE_TYPES, default=list, @@ -764,7 +765,7 @@ def validate_includes(self, value): Requirement(pkg) except ValueError as ve: raise serializers.ValidationError( - _("includes specifier {} is invalid. {}".format(pkg, ve)) + _("includes specifier {} is invalid. {}").format(pkg, ve) ) return value @@ -775,7 +776,7 @@ def validate_excludes(self, value): Requirement(pkg) except ValueError as ve: raise serializers.ValidationError( - _("excludes specifier {} is invalid. {}".format(pkg, ve)) + _("excludes specifier {} is invalid. {}").format(pkg, ve) ) return value diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index 4604a60f9..7c872159d 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -1,6 +1,6 @@ -from gettext import gettext as _ import logging import os +from gettext import gettext as _ from django.core.files import File from packaging.utils import canonicalize_name @@ -10,7 +10,7 @@ from pulp_python.app import models as python_models from pulp_python.app.serializers import PythonPublicationSerializer -from pulp_python.app.utils import write_simple_index, write_simple_detail +from pulp_python.app.utils import write_simple_detail, write_simple_index log = logging.getLogger(__name__) diff --git a/pulp_python/app/tasks/repair.py b/pulp_python/app/tasks/repair.py index fb4d7acf5..ed79988e2 100644 --- a/pulp_python/app/tasks/repair.py +++ b/pulp_python/app/tasks/repair.py @@ -7,6 +7,10 @@ from django.db.models import Prefetch from django.db.models.query import QuerySet + +from pulpcore.plugin.models import ContentArtifact, ProgressReport +from pulpcore.plugin.util import get_domain + from pulp_python.app.models import PythonPackageContent, PythonRepository from pulp_python.app.utils import ( artifact_to_python_content_data, @@ -16,8 +20,6 @@ metadata_content_to_artifact, parse_metadata, ) -from pulpcore.plugin.models import ContentArtifact, ProgressReport -from pulpcore.plugin.util import get_domain log = logging.getLogger(__name__) diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index 1ceaf2072..ee957da98 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -1,11 +1,17 @@ -import logging import asyncio - -from aiohttp import ClientResponseError, ClientError -from lxml.etree import LxmlError -from gettext import gettext as _ +import logging from functools import partial +from gettext import gettext as _ +from urllib.parse import urljoin +from aiohttp import ClientError, ClientResponseError +from bandersnatch.configuration import BandersnatchConfig +from bandersnatch.master import Master +from bandersnatch.mirror import Mirror +from lxml.etree import LxmlError +from packaging.requirements import Requirement +from pypi_attestations import Provenance +from pypi_simple import IndexPage from rest_framework import serializers from pulpcore.plugin.download import HttpDownloader @@ -18,19 +24,11 @@ ) from pulp_python.app.models import ( + PackageProvenance, PythonPackageContent, PythonRemote, - PackageProvenance, ) -from pulp_python.app.utils import parse_metadata, PYPI_LAST_SERIAL, aget_remote_simple_page -from pypi_simple import IndexPage -from pypi_attestations import Provenance - -from bandersnatch.mirror import Mirror -from bandersnatch.master import Master -from bandersnatch.configuration import BandersnatchConfig -from packaging.requirements import Requirement -from urllib.parse import urljoin +from pulp_python.app.utils import PYPI_LAST_SERIAL, aget_remote_simple_page, parse_metadata logger = logging.getLogger(__name__) diff --git a/pulp_python/app/tasks/upload.py b/pulp_python/app/tasks/upload.py index bf98342dc..48faebeb5 100644 --- a/pulp_python/app/tasks/upload.py +++ b/pulp_python/app/tasks/upload.py @@ -1,17 +1,18 @@ import time - from datetime import datetime, timezone -from django.db import transaction + from django.contrib.sessions.models import Session +from django.db import transaction from pydantic import TypeAdapter -from pulpcore.plugin.models import Artifact, CreatedResource, Content, ContentArtifact -from pulpcore.plugin.util import get_domain, get_current_authenticated_user, get_prn -from pulp_python.app.models import PythonPackageContent, PythonRepository, PackageProvenance +from pulpcore.plugin.models import Artifact, Content, ContentArtifact, CreatedResource +from pulpcore.plugin.util import get_current_authenticated_user, get_domain, get_prn + +from pulp_python.app.models import PackageProvenance, PythonPackageContent, PythonRepository from pulp_python.app.provenance import ( + AnyPublisher, Attestation, AttestationBundle, - AnyPublisher, Provenance, verify_provenance, ) diff --git a/pulp_python/app/urls.py b/pulp_python/app/urls.py index 5e3299a96..309900b9d 100644 --- a/pulp_python/app/urls.py +++ b/pulp_python/app/urls.py @@ -2,11 +2,11 @@ from django.urls import path from pulp_python.app.pypi.views import ( - SimpleView, MetadataView, + ProvenanceView, PyPIView, + SimpleView, UploadView, - ProvenanceView, ) if settings.DOMAIN_ENABLED: diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 9c4eb15cd..203a15300 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -1,23 +1,25 @@ import hashlib +import json import logging -import pkginfo import re import shutil import tempfile import zipfile -import json -from aiohttp.client_exceptions import ClientError from collections import defaultdict from datetime import timezone + +import pkginfo +from aiohttp.client_exceptions import ClientError from django.conf import settings from django.db.utils import IntegrityError from jinja2 import Template -from packaging.utils import canonicalize_name from packaging.requirements import Requirement -from packaging.version import parse, InvalidVersion +from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, parse from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage -from pulpcore.plugin.models import Artifact, Remote + from pulpcore.plugin.exceptions import TimeoutException +from pulpcore.plugin.models import Artifact, Remote from pulpcore.plugin.util import get_domain log = logging.getLogger(__name__) diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 83109b36b..4b74e0bd7 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -1,10 +1,11 @@ +from pathlib import Path + from bandersnatch.configuration import BandersnatchConfig from django.db import transaction from django_filters import CharFilter from django_filters.rest_framework import filters as drf_filters from drf_spectacular.utils import extend_schema, extend_schema_view from packaging.utils import canonicalize_name -from pathlib import Path from rest_framework import status from rest_framework.decorators import action from rest_framework.mixins import ( diff --git a/pulp_python/pytest_plugin.py b/pulp_python/pytest_plugin.py index 799a7a2ae..cf4c6c119 100644 --- a/pulp_python/pytest_plugin.py +++ b/pulp_python/pytest_plugin.py @@ -1,16 +1,18 @@ -import pytest -import uuid import subprocess +import uuid + +import pytest from pulpcore.tests.functional.utils import BindingsNamespace + from pulp_python.tests.functional.constants import ( - PYTHON_FIXTURE_URL, - PYTHON_XS_PROJECT_SPECIFIER, PYTHON_EGG_FILENAME, - PYTHON_URL, PYTHON_EGG_URL, - PYTHON_WHEEL_URL, + PYTHON_FIXTURE_URL, + PYTHON_URL, PYTHON_WHEEL_FILENAME, + PYTHON_WHEEL_URL, + PYTHON_XS_PROJECT_SPECIFIER, ) # Bindings API Fixtures diff --git a/pulp_python/tests/functional/api/test_attestations.py b/pulp_python/tests/functional/api/test_attestations.py index 2fb652f24..0737256b7 100644 --- a/pulp_python/tests/functional/api/test_attestations.py +++ b/pulp_python/tests/functional/api/test_attestations.py @@ -1,11 +1,11 @@ -import pytest import json -import requests import shutil import subprocess from pathlib import Path from urllib.parse import urljoin +import pytest +import requests from pypi_simple import PyPISimple from pulpcore.tests.functional.utils import PulpTaskError diff --git a/pulp_python/tests/functional/api/test_blocklist.py b/pulp_python/tests/functional/api/test_blocklist.py index c6c62f859..8aeed2ecd 100644 --- a/pulp_python/tests/functional/api/test_blocklist.py +++ b/pulp_python/tests/functional/api/test_blocklist.py @@ -1,6 +1,7 @@ import pytest from pulpcore.tests.functional.utils import PulpTaskError + from pulp_python.tests.functional.constants import PYTHON_EGG_FILENAME, PYTHON_EGG_URL CONTENT_BODY = {"relative_path": PYTHON_EGG_FILENAME, "file_url": PYTHON_EGG_URL} diff --git a/pulp_python/tests/functional/api/test_consume_content.py b/pulp_python/tests/functional/api/test_consume_content.py index 98ac094ee..56ee21400 100644 --- a/pulp_python/tests/functional/api/test_consume_content.py +++ b/pulp_python/tests/functional/api/test_consume_content.py @@ -1,5 +1,4 @@ import subprocess - from urllib.parse import urlsplit diff --git a/pulp_python/tests/functional/api/test_crud_content_unit.py b/pulp_python/tests/functional/api/test_crud_content_unit.py index fe049b3bd..5bf799563 100644 --- a/pulp_python/tests/functional/api/test_crud_content_unit.py +++ b/pulp_python/tests/functional/api/test_crud_content_unit.py @@ -1,14 +1,15 @@ -import pytest - from urllib.parse import urljoin + +import pytest from pypi_simple import PyPISimple from pulpcore.tests.functional.utils import PulpTaskError + from pulp_python.tests.functional.constants import ( - PYTHON_FIXTURES_URL, - PYTHON_PACKAGE_DATA, PYTHON_EGG_FILENAME, PYTHON_EGG_URL, + PYTHON_FIXTURES_URL, + PYTHON_PACKAGE_DATA, PYTHON_SM_FIXTURE_CHECKSUMS, PYTHON_WHEEL_FILENAME, PYTHON_WHEEL_URL, diff --git a/pulp_python/tests/functional/api/test_crud_publications.py b/pulp_python/tests/functional/api/test_crud_publications.py index b54b8aa9e..e7941ac18 100644 --- a/pulp_python/tests/functional/api/test_crud_publications.py +++ b/pulp_python/tests/functional/api/test_crud_publications.py @@ -1,13 +1,14 @@ -import pytest import random -from pypi_simple import PyPISimple from urllib.parse import urljoin +import pytest +from pypi_simple import PyPISimple + from pulp_python.tests.functional.constants import ( - PYTHON_SM_PROJECT_SPECIFIER, - PYTHON_SM_FIXTURE_RELEASES, - PYTHON_SM_FIXTURE_CHECKSUMS, PYTHON_EGG_FILENAME, + PYTHON_SM_FIXTURE_CHECKSUMS, + PYTHON_SM_FIXTURE_RELEASES, + PYTHON_SM_PROJECT_SPECIFIER, PYTHON_WHEEL_FILENAME, ) from pulp_python.tests.functional.utils import ensure_simple diff --git a/pulp_python/tests/functional/api/test_crud_remotes.py b/pulp_python/tests/functional/api/test_crud_remotes.py index ace928fd3..eaaec3659 100644 --- a/pulp_python/tests/functional/api/test_crud_remotes.py +++ b/pulp_python/tests/functional/api/test_crud_remotes.py @@ -1,11 +1,12 @@ -import pytest import uuid +import pytest + from pulp_python.tests.functional.constants import ( BANDERSNATCH_CONF, DEFAULT_BANDER_REMOTE_BODY, - PYTHON_INVALID_SPECIFIER_NO_NAME, PYTHON_INVALID_SPECIFIER_BAD_VERSION, + PYTHON_INVALID_SPECIFIER_NO_NAME, PYTHON_VALID_SPECIFIER_NO_VERSION, ) diff --git a/pulp_python/tests/functional/api/test_domains.py b/pulp_python/tests/functional/api/test_domains.py index b67be1d54..e349cfea3 100644 --- a/pulp_python/tests/functional/api/test_domains.py +++ b/pulp_python/tests/functional/api/test_domains.py @@ -1,16 +1,17 @@ -import pytest -import uuid import json import subprocess +import uuid +from urllib.parse import urlsplit -from pulpcore.app import settings +import pytest + +from pulpcore.app import settings # noqa: TID251 from pulp_python.tests.functional.constants import ( PYTHON_EGG_FILENAME, - PYTHON_SM_PROJECT_SPECIFIER, PYTHON_SM_PACKAGE_COUNT, + PYTHON_SM_PROJECT_SPECIFIER, ) -from urllib.parse import urlsplit pytestmark = pytest.mark.skipif(not settings.DOMAIN_ENABLED, reason="Domain not enabled") diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index 11ca94494..c9820f114 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -1,10 +1,10 @@ import pytest from pulp_python.tests.functional.constants import ( - PYTHON_MD_PROJECT_SPECIFIER, + PYTHON_LG_PACKAGE_COUNT, PYTHON_LG_PROJECT_SPECIFIER, PYTHON_MD_PACKAGE_COUNT, - PYTHON_LG_PACKAGE_COUNT, + PYTHON_MD_PROJECT_SPECIFIER, ) diff --git a/pulp_python/tests/functional/api/test_export_import.py b/pulp_python/tests/functional/api/test_export_import.py index 8eb623eff..9c49cc3ed 100644 --- a/pulp_python/tests/functional/api/test_export_import.py +++ b/pulp_python/tests/functional/api/test_export_import.py @@ -5,13 +5,15 @@ the case. """ -import pytest import uuid -from pulpcore.app import settings +import pytest + +from pulpcore.app import settings # noqa: TID251 + from pulp_python.tests.functional.constants import ( - PYTHON_XS_PROJECT_SPECIFIER, PYTHON_SM_PROJECT_SPECIFIER, + PYTHON_XS_PROJECT_SPECIFIER, ) pytestmark = [ diff --git a/pulp_python/tests/functional/api/test_full_mirror.py b/pulp_python/tests/functional/api/test_full_mirror.py index a20b48c00..3cc842e4c 100644 --- a/pulp_python/tests/functional/api/test_full_mirror.py +++ b/pulp_python/tests/functional/api/test_full_mirror.py @@ -1,20 +1,20 @@ +import subprocess +from hashlib import sha256 +from random import sample +from urllib.parse import urljoin, urlsplit, urlunsplit + import pytest import requests -import subprocess +from packaging.version import parse +from pypi_simple import ProjectPage from pulp_python.tests.functional.constants import ( PYPI_URL, - PYTHON_XS_FIXTURE_CHECKSUMS, - PYTHON_SM_PROJECT_SPECIFIER, PYTHON_SM_FIXTURE_RELEASES, + PYTHON_SM_PROJECT_SPECIFIER, + PYTHON_XS_FIXTURE_CHECKSUMS, ) -from pypi_simple import ProjectPage -from packaging.version import parse -from urllib.parse import urljoin, urlsplit, urlunsplit -from random import sample -from hashlib import sha256 - def test_pull_through_install( python_bindings, python_remote_factory, python_distribution_factory, delete_orphans_pre diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index 2cbdc87a2..4934d3a98 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -1,15 +1,15 @@ -import pytest -import requests import subprocess - from urllib.parse import urljoin +import pytest +import requests + from pulp_python.tests.functional.constants import ( PYPI_SERIAL_CONSTANT, - PYTHON_MD_PROJECT_SPECIFIER, - PYTHON_MD_PYPI_SUMMARY, PYTHON_EGG_FILENAME, PYTHON_EGG_SHA256, + PYTHON_MD_PROJECT_SPECIFIER, + PYTHON_MD_PYPI_SUMMARY, PYTHON_WHEEL_FILENAME, PYTHON_WHEEL_SHA256, SHELF_PYTHON_JSON, diff --git a/pulp_python/tests/functional/api/test_pypi_simple_api.py b/pulp_python/tests/functional/api/test_pypi_simple_api.py index 621ed9a6b..c9e6b7dd8 100644 --- a/pulp_python/tests/functional/api/test_pypi_simple_api.py +++ b/pulp_python/tests/functional/api/test_pypi_simple_api.py @@ -69,7 +69,7 @@ def test_simple_html_detail_api( monitor_task(python_bindings.RepositoriesPythonApi.modify(repo.pulp_href, body).task) distro = python_distribution_factory(repository=repo) - url = f'{urljoin(distro.base_url, "simple/")}twine' + url = f"{urljoin(distro.base_url, 'simple/')}twine" headers = {"Accept": PYPI_SIMPLE_V1_HTML} response = requests.get(url, headers=headers) @@ -125,7 +125,7 @@ def test_simple_json_detail_api( monitor_task(python_bindings.RepositoriesPythonApi.modify(repo.pulp_href, body).task) distro = python_distribution_factory(repository=repo) - url = f'{urljoin(distro.base_url, "simple/")}twine' + url = f"{urljoin(distro.base_url, 'simple/')}twine" headers = {"Accept": PYPI_SIMPLE_V1_JSON} response = requests.get(url, headers=headers) diff --git a/pulp_python/tests/functional/api/test_rbac.py b/pulp_python/tests/functional/api/test_rbac.py index 2c46977e0..4990b1430 100644 --- a/pulp_python/tests/functional/api/test_rbac.py +++ b/pulp_python/tests/functional/api/test_rbac.py @@ -1,6 +1,7 @@ -import pytest import uuid +import pytest + from pulp_python.tests.functional.constants import ( PYTHON_EGG_FILENAME, PYTHON_EGG_SHA256, @@ -44,9 +45,9 @@ def _try_action(user, client, action, outcome, *args, **kwargs): except python_bindings.module.ApiException as e: assert e.status == outcome, f"{e}" else: - assert ( - status_code == outcome - ), f"User performed {action} when they shouldn't been able to" + assert status_code == outcome, ( + f"User performed {action} when they shouldn't been able to" + ) return data return _try_action diff --git a/pulp_python/tests/functional/api/test_repair.py b/pulp_python/tests/functional/api/test_repair.py index 102f868e8..3a32f5a29 100644 --- a/pulp_python/tests/functional/api/test_repair.py +++ b/pulp_python/tests/functional/api/test_repair.py @@ -1,7 +1,8 @@ -import pytest import subprocess from urllib.parse import urljoin +import pytest + from pulp_python.tests.functional.constants import ( PYTHON_EGG_FILENAME, PYTHON_FIXTURES_URL, diff --git a/pulp_python/tests/functional/api/test_sync.py b/pulp_python/tests/functional/api/test_sync.py index a11806a5b..83b9c9c2a 100644 --- a/pulp_python/tests/functional/api/test_sync.py +++ b/pulp_python/tests/functional/api/test_sync.py @@ -1,21 +1,21 @@ import pytest from pulp_python.tests.functional.constants import ( - PYTHON_XS_PACKAGE_COUNT, - PYTHON_PRERELEASE_TEST_SPECIFIER, - PYTHON_WITH_PRERELEASE_COUNT, - PYTHON_WITHOUT_PRERELEASE_COUNT, - PYTHON_XS_PROJECT_SPECIFIER, - PYTHON_MD_PROJECT_SPECIFIER, + DJANGO_LATEST_3, + PYTHON_LG_FIXTURE_COUNTS, + PYTHON_LG_PACKAGE_COUNT, + PYTHON_LG_PROJECT_SPECIFIER, PYTHON_MD_PACKAGE_COUNT, - PYTHON_SM_PROJECT_SPECIFIER, + PYTHON_MD_PROJECT_SPECIFIER, + PYTHON_PRERELEASE_TEST_SPECIFIER, PYTHON_SM_PACKAGE_COUNT, + PYTHON_SM_PROJECT_SPECIFIER, PYTHON_UNAVAILABLE_PACKAGE_COUNT, PYTHON_UNAVAILABLE_PROJECT_SPECIFIER, - PYTHON_LG_PROJECT_SPECIFIER, - PYTHON_LG_PACKAGE_COUNT, - PYTHON_LG_FIXTURE_COUNTS, - DJANGO_LATEST_3, + PYTHON_WITH_PRERELEASE_COUNT, + PYTHON_WITHOUT_PRERELEASE_COUNT, + PYTHON_XS_PACKAGE_COUNT, + PYTHON_XS_PROJECT_SPECIFIER, SCIPY_COUNTS, ) from pulp_python.tests.functional.utils import ensure_metadata diff --git a/pulp_python/tests/functional/api/test_upload.py b/pulp_python/tests/functional/api/test_upload.py index 43c65cdfe..71ba32a79 100644 --- a/pulp_python/tests/functional/api/test_upload.py +++ b/pulp_python/tests/functional/api/test_upload.py @@ -1,16 +1,18 @@ +from urllib.parse import urljoin + import pytest import requests + from pulp_python.tests.functional.constants import ( PYTHON_EGG_FILENAME, + PYTHON_EGG_SHA256, PYTHON_EGG_URL, PYTHON_FIXTURES_URL, PYTHON_WHEEL_FILENAME, - PYTHON_WHEEL_URL, - PYTHON_EGG_SHA256, PYTHON_WHEEL_SHA256, + PYTHON_WHEEL_URL, ) from pulp_python.tests.functional.utils import ensure_metadata -from urllib.parse import urljoin @pytest.mark.parametrize( diff --git a/pulp_python/tests/functional/utils.py b/pulp_python/tests/functional/utils.py index 9d672b80c..09a19c154 100644 --- a/pulp_python/tests/functional/utils.py +++ b/pulp_python/tests/functional/utils.py @@ -1,6 +1,6 @@ -import requests - from urllib.parse import urljoin + +import requests from lxml import html diff --git a/pyproject.toml b/pyproject.toml index 2fe313e4d..f92628253 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,5 +124,33 @@ replace = "version = \"{new_version}\"" filename = "./pyproject.toml" search = "version = \"{current_version}\"" replace = "version = \"{new_version}\"" + + [tool.black] line-length = 100 + +[tool.ruff] +# This section is managed by the plugin template. Do not edit manually. +line-length = 100 +extend-exclude = [ + "docs/**", + "**/migrations/*.py", +] + +[tool.ruff.lint] +# This section is managed by the plugin template. Do not edit manually. +extend-select = [ + "I", + "INT", + "TID", + "T10", +] + +[tool.ruff.lint.flake8-tidy-imports.banned-api] +# This section is managed by the plugin template. Do not edit manually. +"pulpcore.app".msg = "The 'pulpcore' apis must only be consumed via 'pulpcore.plugin'." + +[tool.ruff.lint.isort] +# This section is managed by the plugin template. Do not edit manually. +sections = { second-party = ["pulpcore"] } +section-order = ["future", "standard-library", "third-party", "second-party", "first-party", "local-folder"] diff --git a/template_config.yml b/template_config.yml index 3ba34765d..04a38e3fa 100644 --- a/template_config.yml +++ b/template_config.yml @@ -6,9 +6,7 @@ # After editing this file please always reapply the plugin template before committing any changes. --- -black: true check_commit_message: true -check_gettext: true check_manifest: true check_stray_pulpcore_imports: true ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" @@ -23,10 +21,9 @@ deploy_to_pypi: true disabled_redis_runners: [] docker_fixtures: false extra_files: [] -flake8: true -flake8_ignore: [] github_org: "pulp" latest_release_branch: "3.29" +lint_ignore: [] lint_requirements: true os_required_packages: [] parallel_test_workers: 8 @@ -83,7 +80,6 @@ pulp_settings_s3: BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" api_root: "/rerouted/djnd/" domain_enabled: true -pydocstyle: true release_email: "pulp-infra@redhat.com" release_user: "pulpbot" stalebot: true