diff --git a/.ci/ansible/filter/repr.py b/.ci/ansible/filter/repr.py index 8455c3442..c8c1678dd 100644 --- a/.ci/ansible/filter/repr.py +++ b/.ci/ansible/filter/repr.py @@ -1,4 +1,5 @@ from __future__ import absolute_import, division, print_function + from packaging.version import parse as parse_version __metaclass__ = type diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 66c494e97..83e197aa7 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -7,11 +7,12 @@ import argparse import fileinput -import urllib.request import sys +import urllib.request + +import yaml from packaging.requirements import Requirement from packaging.version import Version -import yaml try: import tomllib diff --git a/.ci/scripts/check_gettext.sh b/.ci/scripts/check_gettext.sh deleted file mode 100755 index 39bdeb04f..000000000 --- a/.ci/scripts/check_gettext.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../.. - -set -uv - -MATCHES=$(grep -n -r --include \*.py "_(f") - -if [ $? -ne 1 ]; then - printf "\nERROR: Detected mix of f-strings and gettext:\n" - echo "$MATCHES" - exit 1 -fi diff --git a/.ci/scripts/check_pulpcore_imports.sh b/.ci/scripts/check_pulpcore_imports.sh index 5d9c6f481..e1867c625 100755 --- a/.ci/scripts/check_pulpcore_imports.sh +++ b/.ci/scripts/check_pulpcore_imports.sh @@ -10,10 +10,10 @@ # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../.. -set -uv +set -u # check for imports not from pulpcore.plugin. exclude tests -MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin") +MATCHES="$(grep -n -r --include \*.py "from pulpcore.*import" pulp_python | grep -v "tests\|plugin")" if [ $? -ne 1 ]; then printf "\nERROR: Detected bad imports from pulpcore:\n" diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 6e0799952..32e2a2ffd 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -9,16 +9,16 @@ # /// import argparse -import re import os +import re import sys -import tomllib import typing as t from pathlib import Path +import tomllib import yaml -from packaging.version import Version from git import Repo +from packaging.version import Version RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" Y_CHANGELOG_EXTS = [".feature"] @@ -157,9 +157,9 @@ def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int: if reasons: curr_version = Version(last_tag) - assert curr_version.base_version.startswith( - branch - ), "Current-version has to belong to the current branch!" + assert curr_version.base_version.startswith(branch), ( + "Current-version has to belong to the current branch!" + ) next_version = Version(f"{branch}.{curr_version.micro + 1}") print( f"A Z-release is needed for {branch}, " diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index cf9efbe97..71253cb3e 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,15 +5,14 @@ # # For more info visit https://github.com/pulp/plugin_template -import tomllib import warnings -from packaging.requirements import Requirement +import tomllib +from packaging.requirements import Requirement CHECK_MATRIX = [ ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), - ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), ("doc_requirements.txt", False, True, False), ("lint_requirements.txt", False, True, True), diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index fbb5d59d0..5d88b5c1f 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -18,14 +18,13 @@ import json import os import re -import tomllib import urllib.request from pathlib import Path +import tomllib from git import GitCommandError, Repo from packaging.version import parse as parse_version - PYPI_PROJECT = "pulp_python" # Read Towncrier settings diff --git a/.ci/scripts/pr_labels.py b/.ci/scripts/pr_labels.py index 0c478a212..4f801c39e 100755 --- a/.ci/scripts/pr_labels.py +++ b/.ci/scripts/pr_labels.py @@ -4,9 +4,9 @@ import re import sys -import tomllib from pathlib import Path +import tomllib from git import Repo diff --git a/.ci/scripts/schema.py b/.ci/scripts/schema.py index 9f56caa66..9c8e11b2e 100644 --- a/.ci/scripts/schema.py +++ b/.ci/scripts/schema.py @@ -7,7 +7,9 @@ But some pulp paths start with curly brackets e.g. {artifact_href} This script modifies drf-spectacular schema validation to accept slashes and curly brackets. """ + import json + from drf_spectacular.validation import JSON_SCHEMA_SPEC_PATH with open(JSON_SCHEMA_SPEC_PATH) as fh: diff --git a/.ci/scripts/skip_tests.py b/.ci/scripts/skip_tests.py index a68d000d6..380a3da9f 100755 --- a/.ci/scripts/skip_tests.py +++ b/.ci/scripts/skip_tests.py @@ -15,12 +15,13 @@ *: Error """ -import sys +import argparse import os import re -import git +import sys import textwrap -import argparse + +import git DOC_PATTERNS = [ r"^docs/", diff --git a/.ci/scripts/update_github.py b/.ci/scripts/update_github.py index b298ad836..f5e81a5fb 100755 --- a/.ci/scripts/update_github.py +++ b/.ci/scripts/update_github.py @@ -6,6 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template import os + from github import Github g = Github(os.environ.get("GITHUB_TOKEN")) diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index a4dc9004a..96b84172f 100644 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -5,10 +5,10 @@ import re import subprocess import sys -import tomllib -import yaml from pathlib import Path +import tomllib +import yaml from github import Github diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 9e00b3faa..000000000 --- a/.flake8 +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template -[flake8] -exclude = ./docs/*,*/migrations/* -per-file-ignores = */__init__.py: F401 - -ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402,F824 -max-line-length = 100 - -# Flake8 builtin codes -# -------------------- -# E203: no whitespace around ':'. disabled until https://github.com/PyCQA/pycodestyle/issues/373 is fixed -# W503: This enforces operators before line breaks which is not pep8 or black compatible. -# F824: 'nonlocal' is unused: name is never assigned in scope - -# Flake8-quotes extension codes -# ----------------------------- -# Q000: double or single quotes only, default is double (don't want to enforce this) -# Q003: Change outer quotes to avoid escaping inner quotes - -# Flake8-docstring extension codes -# -------------------------------- -# D100: missing docstring in public module -# D104: missing docstring in public package -# D106: missing docstring in public nested class (complains about "class Meta:" and documenting those is silly) -# D200: one-line docstring should fit on one line with quotes -# D205: 1 blank line required between summary line and description -# D400: First line should end with a period -# D401: first line should be imperative (nitpicky) -# D402: first line should not be the function’s “signature” (false positives) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d07a39a08..8da63002e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -36,14 +36,13 @@ jobs: run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows - # Lint code. - - name: "Run flake8" + - name: "Check formating" run: | - flake8 + ruff format --check --diff - - name: "Check for common gettext problems" + - name: "Lint code" run: | - sh .ci/scripts/check_gettext.sh + ruff check - name: "Run extra lint checks" run: | diff --git a/.github/workflows/scripts/stage-changelog-for-default-branch.py b/.github/workflows/scripts/stage-changelog-for-default-branch.py index 3950d7f9c..255c58424 100755 --- a/.github/workflows/scripts/stage-changelog-for-default-branch.py +++ b/.github/workflows/scripts/stage-changelog-for-default-branch.py @@ -12,7 +12,6 @@ from git import Repo from git.exc import GitCommandError - helper = textwrap.dedent( """\ Stage the changelog for a release on main branch. diff --git a/.github/workflows/scripts/update_backport_labels.py b/.github/workflows/scripts/update_backport_labels.py index 967984a42..063314b6c 100755 --- a/.github/workflows/scripts/update_backport_labels.py +++ b/.github/workflows/scripts/update_backport_labels.py @@ -5,10 +5,11 @@ # # For more info visit https://github.com/pulp/plugin_template +import os +import random + import requests import yaml -import random -import os def random_color(): diff --git a/dev_requirements.txt b/dev_requirements.txt deleted file mode 100644 index 0d7d2e745..000000000 --- a/dev_requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -black -check-manifest -flake8 -flake8-docstrings -flake8-tuple -flake8-quotes -# pin pydocstyle until https://gitlab.com/pycqa/flake8-docstrings/issues/36 is resolved -pydocstyle<4 -requests diff --git a/lint_requirements.txt b/lint_requirements.txt index a29362097..fd592d7d8 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -7,6 +7,6 @@ bump-my-version check-manifest -flake8 packaging +ruff yamllint diff --git a/pulp_python/__init__.py b/pulp_python/__init__.py index 5d57d16b3..50f87cf34 100644 --- a/pulp_python/__init__.py +++ b/pulp_python/__init__.py @@ -1 +1 @@ -default_app_config = 'pulp_python.app.PulpPythonPluginAppConfig' +default_app_config = "pulp_python.app.PulpPythonPluginAppConfig" diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index dea802654..9e8f18f32 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -1,6 +1,8 @@ +from gettext import gettext as _ + from django.db.models.signals import post_migrate + from pulpcore.plugin import PulpPluginAppConfig -from gettext import gettext as _ class PulpPythonPluginAppConfig(PulpPluginAppConfig): @@ -26,7 +28,7 @@ def ready(self): # TODO: Remove this when https://github.com/pulp/pulpcore/issues/5500 is resolved def _populate_pypi_access_policies(sender, apps, verbosity, **kwargs): - from pulp_python.app.pypi.views import PyPIView, SimpleView, UploadView, MetadataView + from pulp_python.app.pypi.views import MetadataView, PyPIView, SimpleView, UploadView try: AccessPolicy = apps.get_model("core", "AccessPolicy") diff --git a/pulp_python/app/global_access_conditions.py b/pulp_python/app/global_access_conditions.py index a2a5ee617..8abb14f18 100644 --- a/pulp_python/app/global_access_conditions.py +++ b/pulp_python/app/global_access_conditions.py @@ -1,6 +1,5 @@ from django.conf import settings - # Access Condition methods that can be used with PyPI access policies diff --git a/pulp_python/app/management/commands/repair-python-metadata.py b/pulp_python/app/management/commands/repair-python-metadata.py index 4582f3282..e97c13d0c 100644 --- a/pulp_python/app/management/commands/repair-python-metadata.py +++ b/pulp_python/app/management/commands/repair-python-metadata.py @@ -1,11 +1,12 @@ -import re import os -from django.core.management import BaseCommand, CommandError +import re from gettext import gettext as _ from django.conf import settings +from django.core.management import BaseCommand, CommandError from pulpcore.plugin.util import extract_pk + from pulp_python.app.models import PythonPackageContent, PythonRepository from pulp_python.app.utils import artifact_to_python_content_data @@ -23,7 +24,7 @@ def repair_metadata(content): batch = [] set_of_update_fields = set() total_repaired = 0 - for package in immediate_content.prefetch_related('_artifacts').iterator(chunk_size=1000): + for package in immediate_content.prefetch_related("_artifacts").iterator(chunk_size=1000): new_data = artifact_to_python_content_data( package.filename, package._artifacts.get(), package.pulp_domain ) @@ -55,7 +56,7 @@ def href_prn_list_handler(value): (?:{settings.API_ROOT}(?:[-_a-zA-Z0-9]+/)?api/v3/repositories/python/python/[-a-f0-9]+/) |(?:prn:python\.pythonrepository:[-a-f0-9]+) """, - re.VERBOSE + re.VERBOSE, ) values = [] for v in value.split(","): diff --git a/pulp_python/app/modelresource.py b/pulp_python/app/modelresource.py index 6533c8810..bc5f41781 100644 --- a/pulp_python/app/modelresource.py +++ b/pulp_python/app/modelresource.py @@ -1,6 +1,7 @@ from pulpcore.plugin.importexport import BaseContentResource from pulpcore.plugin.modelresources import RepositoryResource from pulpcore.plugin.util import get_domain + from pulp_python.app.models import ( PythonPackageContent, PythonRepository, diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index c326a54a4..568aae8a6 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -1,30 +1,31 @@ from logging import getLogger +from pathlib import PurePath from aiohttp.web import json_response +from django.conf import settings from django.contrib.postgres.fields import ArrayField from django.core.exceptions import ObjectDoesNotExist from django.db import models -from django.conf import settings + from pulpcore.plugin.models import ( AutoAddObjPermsMixin, Content, - Publication, Distribution, + Publication, Remote, Repository, ) +from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version from pulpcore.plugin.responses import ArtifactResponse +from pulpcore.plugin.util import get_domain, get_domain_pk -from pathlib import PurePath from .utils import ( + PYPI_LAST_SERIAL, + PYPI_SERIAL_CONSTANT, artifact_to_python_content_data, canonicalize_name, python_content_to_json, - PYPI_LAST_SERIAL, - PYPI_SERIAL_CONSTANT, ) -from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version -from pulpcore.plugin.util import get_domain_pk, get_domain log = getLogger(__name__) diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py index dc99fc802..dac9624fc 100644 --- a/pulp_python/app/pypi/serializers.py +++ b/pulp_python/app/pypi/serializers.py @@ -1,11 +1,13 @@ import logging from gettext import gettext as _ +from django.db.utils import IntegrityError from rest_framework import serializers -from pulp_python.app.utils import DIST_EXTENSIONS + from pulpcore.plugin.models import Artifact from pulpcore.plugin.util import get_domain -from django.db.utils import IntegrityError + +from pulp_python.app.utils import DIST_EXTENSIONS log = logging.getLogger(__name__) @@ -46,7 +48,7 @@ class PackageUploadSerializer(serializers.Serializer): action = serializers.CharField( help_text=_("Defaults to `file_upload`, don't change it or request will fail!"), default="file_upload", - source=":action" + source=":action", ) sha256_digest = serializers.CharField( help_text=_("SHA256 of package to validate upload integrity."), @@ -59,17 +61,17 @@ def validate(self, data): """Validates the request.""" action = data.get(":action") if action != "file_upload": - raise serializers.ValidationError( - _("We do not support the :action {}").format(action) - ) + raise serializers.ValidationError(_("We do not support the :action {}").format(action)) file = data.get("content") for ext, packagetype in DIST_EXTENSIONS.items(): if file.name.endswith(ext): break else: - raise serializers.ValidationError(_( - "Extension on {} is not a valid python extension " - "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(file.name) + raise serializers.ValidationError( + _( + "Extension on {} is not a valid python extension " + "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)" + ).format(file.name) ) sha256 = data.get("sha256_digest") digests = {"sha256": sha256} if sha256 else None diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index b6210e2b4..9e03745e1 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -1,54 +1,53 @@ import logging -import requests - -from rest_framework.viewsets import ViewSet -from rest_framework.response import Response -from django.core.exceptions import ObjectDoesNotExist -from django.shortcuts import redirect -from datetime import datetime, timezone, timedelta +from datetime import datetime, timedelta, timezone +from itertools import chain +from pathlib import PurePath +from urllib.parse import urljoin, urlparse, urlunsplit -from rest_framework.reverse import reverse +import requests from django.contrib.sessions.models import Session +from django.core.exceptions import ObjectDoesNotExist from django.db import transaction from django.db.utils import DatabaseError from django.http.response import ( Http404, - HttpResponseForbidden, HttpResponseBadRequest, - StreamingHttpResponse + HttpResponseForbidden, + StreamingHttpResponse, ) +from django.shortcuts import redirect from drf_spectacular.utils import extend_schema from dynaconf import settings -from itertools import chain from packaging.utils import canonicalize_name -from urllib.parse import urljoin, urlparse, urlunsplit -from pathlib import PurePath from pypi_simple import parse_links_stream_response +from rest_framework.response import Response +from rest_framework.reverse import reverse +from rest_framework.viewsets import ViewSet -from pulpcore.plugin.viewsets import OperationPostponedResponse from pulpcore.plugin.tasking import dispatch from pulpcore.plugin.util import get_domain +from pulpcore.plugin.viewsets import OperationPostponedResponse + +from pulp_python.app import tasks from pulp_python.app.models import ( PythonDistribution, PythonPackageContent, PythonPublication, ) from pulp_python.app.pypi.serializers import ( - SummarySerializer, PackageMetadataSerializer, PackageUploadSerializer, - PackageUploadTaskSerializer + PackageUploadTaskSerializer, + SummarySerializer, ) from pulp_python.app.utils import ( - write_simple_index, - write_simple_detail, - python_content_to_json, PYPI_LAST_SERIAL, PYPI_SERIAL_CONSTANT, + python_content_to_json, + write_simple_detail, + write_simple_index, ) -from pulp_python.app import tasks - log = logging.getLogger(__name__) ORIGIN_HOST = settings.CONTENT_ORIGIN if settings.CONTENT_ORIGIN else settings.PYPI_API_HOSTNAME @@ -159,10 +158,15 @@ def upload(self, request, path): if settings.PYTHON_GROUP_UPLOADS: return self.upload_package_group(repo, artifact, filename, request.session) - result = dispatch(tasks.upload, exclusive_resources=[artifact, repo], - kwargs={"artifact_sha256": artifact.sha256, - "filename": filename, - "repository_pk": str(repo.pk)}) + result = dispatch( + tasks.upload, + exclusive_resources=[artifact, repo], + kwargs={ + "artifact_sha256": artifact.sha256, + "filename": filename, + "repository_pk": str(repo.pk), + }, + ) return OperationPostponedResponse(result, request) def upload_package_group(self, repo, artifact, filename, session): @@ -176,10 +180,10 @@ def upload_package_group(self, repo, artifact, filename, session): try: with transaction.atomic(): sq.first() - current_start = datetime.fromisoformat(session['start']) + current_start = datetime.fromisoformat(session["start"]) if current_start >= datetime.now(tz=timezone.utc): - session['artifacts'].append((str(artifact.sha256), filename)) - session['start'] = str(start_time) + session["artifacts"].append((str(artifact.sha256), filename)) + session["start"] = str(start_time) session.modified = False session.save() else: @@ -192,14 +196,19 @@ def upload_package_group(self, repo, artifact, filename, session): def create_group_upload_task(self, cur_session, repository, artifact, filename, start_time): """Creates the actual task that adds the packages to the index.""" - cur_session['start'] = str(start_time) - cur_session['artifacts'] = [(str(artifact.sha256), filename)] + cur_session["start"] = str(start_time) + cur_session["artifacts"] = [(str(artifact.sha256), filename)] cur_session.modified = False cur_session.save() - result = dispatch(tasks.upload_group, exclusive_resources=[artifact, repository], - kwargs={"session_pk": str(cur_session.session_key), - "repository_pk": str(repository.pk)}) - return reverse('tasks-detail', args=[result.pk], request=None) + result = dispatch( + tasks.upload_group, + exclusive_resources=[artifact, repository], + kwargs={ + "session_pk": str(cur_session.session_key), + "repository_pk": str(repository.pk), + }, + ) + return reverse("tasks-detail", args=[result.pk], request=None) class SimpleView(PackageUploadMixin, ViewSet): @@ -227,21 +236,22 @@ def list(self, request, path): """Gets the simple api html page for the index.""" repo_version, content = self.get_rvc() if self.should_redirect(repo_version=repo_version): - return redirect(urljoin(self.base_content_url, f'{path}/simple/')) - names = content.order_by('name').values_list('name', flat=True).distinct().iterator() + return redirect(urljoin(self.base_content_url, f"{path}/simple/")) + names = content.order_by("name").values_list("name", flat=True).distinct().iterator() return StreamingHttpResponse(write_simple_index(names, streamed=True)) def pull_through_package_simple(self, package, path, remote): """Gets the package's simple page from remote.""" + def parse_url(link): parsed = urlparse(link.url) - digest, _, value = parsed.fragment.partition('=') + digest, _, value = parsed.fragment.partition("=") stripped_url = urlunsplit(chain(parsed[:3], ("", ""))) - redirect = f'{path}/{link.text}?redirect={stripped_url}' + redirect = f"{path}/{link.text}?redirect={stripped_url}" d_url = urljoin(self.base_content_url, redirect) - return link.text, d_url, value if digest == 'sha256' else '' + return link.text, d_url, value if digest == "sha256" else "" - url = remote.get_remote_artifact_url(f'simple/{package}/') + url = remote.get_remote_artifact_url(f"simple/{package}/") kwargs = {} if proxy_url := remote.proxy_url: if remote.proxy_username or remote.proxy_password: @@ -265,10 +275,10 @@ def retrieve(self, request, path, package): if not repo_ver or not content.filter(name__normalize=normalized).exists(): return self.pull_through_package_simple(normalized, path, self.distribution.remote) if self.should_redirect(repo_version=repo_ver): - return redirect(urljoin(self.base_content_url, f'{path}/simple/{normalized}/')) + return redirect(urljoin(self.base_content_url, f"{path}/simple/{normalized}/")) packages = ( content.filter(name__normalize=normalized) - .values_list('filename', 'sha256', 'name') + .values_list("filename", "sha256", "name") .iterator() ) try: @@ -278,12 +288,14 @@ def retrieve(self, request, path, package): else: packages = chain([present], packages) name = present[2] - releases = ((f, urljoin(self.base_content_url, f'{path}/{f}'), d) for f, d, _ in packages) + releases = ((f, urljoin(self.base_content_url, f"{path}/{f}"), d) for f, d, _ in packages) return StreamingHttpResponse(write_simple_detail(name, releases, streamed=True)) - @extend_schema(request=PackageUploadSerializer, - responses={200: PackageUploadTaskSerializer}, - summary="Upload a package") + @extend_schema( + request=PackageUploadSerializer, + responses={200: PackageUploadTaskSerializer}, + summary="Upload a package", + ) def create(self, request, path): """ Upload package to the index. @@ -308,9 +320,11 @@ class MetadataView(PyPIMixin, ViewSet): ], } - @extend_schema(tags=["Pypi: Metadata"], - responses={200: PackageMetadataSerializer}, - summary="Get package metadata") + @extend_schema( + tags=["Pypi: Metadata"], + responses={200: PackageMetadataSerializer}, + summary="Get package metadata", + ) def retrieve(self, request, path, meta): """ Retrieves the package's core-metadata specified by @@ -356,8 +370,7 @@ class PyPIView(PyPIMixin, ViewSet): ], } - @extend_schema(responses={200: SummarySerializer}, - summary="Get index summary") + @extend_schema(responses={200: SummarySerializer}, summary="Get index summary") def retrieve(self, request, path): """Gets package summary stats of index.""" repo_ver, content = self.get_rvc() @@ -383,9 +396,11 @@ class UploadView(PackageUploadMixin, ViewSet): ], } - @extend_schema(request=PackageUploadSerializer, - responses={200: PackageUploadTaskSerializer}, - summary="Upload a package") + @extend_schema( + request=PackageUploadSerializer, + responses={200: PackageUploadTaskSerializer}, + summary="Upload a package", + ) def create(self, request, path): """ Upload package to the index. diff --git a/pulp_python/app/replica.py b/pulp_python/app/replica.py index 5cf015fea..aa2c25d29 100644 --- a/pulp_python/app/replica.py +++ b/pulp_python/app/replica.py @@ -1,10 +1,11 @@ -from pulpcore.plugin.replica import Replicator - from pulp_glue.python.context import ( PulpPythonDistributionContext, PulpPythonPublicationContext, PulpPythonRepositoryContext, ) + +from pulpcore.plugin.replica import Replicator + from pulp_python.app.models import PythonDistribution, PythonRemote, PythonRepository from pulp_python.app.tasks import sync as python_sync diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index dca90d27e..82c044f0a 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -1,4 +1,5 @@ from gettext import gettext as _ + from django.conf import settings from packaging.requirements import Requirement from rest_framework import serializers @@ -44,15 +45,14 @@ class PythonDistributionSerializer(core_serializers.DistributionSerializer): ) base_url = serializers.SerializerMethodField(read_only=True) allow_uploads = serializers.BooleanField( - default=True, - help_text=_("Allow packages to be uploaded to this index.") + default=True, help_text=_("Allow packages to be uploaded to this index.") ) remote = core_serializers.DetailRelatedField( required=False, - help_text=_('Remote that can be used to fetch content when using pull-through caching.'), + help_text=_("Remote that can be used to fetch content when using pull-through caching."), view_name_pattern=r"remotes(-.*/.*)?-detail", queryset=core_models.Remote.objects.all(), - allow_null=True + allow_null=True, ) def get_base_url(self, obj): @@ -63,7 +63,9 @@ def get_base_url(self, obj): class Meta: fields = core_serializers.DistributionSerializer.Meta.fields + ( - 'publication', "allow_uploads", "remote" + "publication", + "allow_uploads", + "remote", ) model = python_models.PythonDistribution @@ -74,126 +76,162 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa """ filename = serializers.CharField( - help_text=_('The name of the distribution package, usually of the format:' - ' {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}' - '-{platform tag}.{packagetype}'), + help_text=_( + "The name of the distribution package, usually of the format:" + " {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}" + "-{platform tag}.{packagetype}" + ), read_only=True, ) packagetype = serializers.CharField( - help_text=_('The type of the distribution package ' - '(e.g. sdist, bdist_wheel, bdist_egg, etc)'), + help_text=_( + "The type of the distribution package (e.g. sdist, bdist_wheel, bdist_egg, etc)" + ), read_only=True, ) name = serializers.CharField( - help_text=_('The name of the python project.'), + help_text=_("The name of the python project."), read_only=True, ) version = serializers.CharField( - help_text=_('The packages version number.'), + help_text=_("The packages version number."), read_only=True, ) sha256 = serializers.CharField( - default='', - help_text=_('The SHA256 digest of this package.'), + default="", + help_text=_("The SHA256 digest of this package."), ) metadata_version = serializers.CharField( - help_text=_('Version of the file format'), + help_text=_("Version of the file format"), read_only=True, ) summary = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A one-line summary of what the package does.') + required=False, + allow_blank=True, + help_text=_("A one-line summary of what the package does."), ) description = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A longer description of the package that can run to several paragraphs.') + required=False, + allow_blank=True, + help_text=_("A longer description of the package that can run to several paragraphs."), ) description_content_type = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A string stating the markup syntax (if any) used in the distribution’s' - ' description, so that tools can intelligently render the description.') + required=False, + allow_blank=True, + help_text=_( + "A string stating the markup syntax (if any) used in the distribution’s" + " description, so that tools can intelligently render the description." + ), ) keywords = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Additional keywords to be used to assist searching for the ' - 'package in a larger catalog.') + required=False, + allow_blank=True, + help_text=_( + "Additional keywords to be used to assist searching for the " + "package in a larger catalog." + ), ) home_page = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The URL for the package\'s home page.') + required=False, allow_blank=True, help_text=_("The URL for the package's home page.") ) download_url = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Legacy field denoting the URL from which this package can be downloaded.') + required=False, + allow_blank=True, + help_text=_("Legacy field denoting the URL from which this package can be downloaded."), ) author = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Text containing the author\'s name. Contact information can also be added,' - ' separated with newlines.') + required=False, + allow_blank=True, + help_text=_( + "Text containing the author's name. Contact information can also be added," + " separated with newlines." + ), ) author_email = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The author\'s e-mail address. ') + required=False, allow_blank=True, help_text=_("The author's e-mail address. ") ) maintainer = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The maintainer\'s name at a minimum; ' - 'additional contact information may be provided.') + required=False, + allow_blank=True, + help_text=_( + "The maintainer's name at a minimum; additional contact information may be provided." + ), ) maintainer_email = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The maintainer\'s e-mail address.') + required=False, allow_blank=True, help_text=_("The maintainer's e-mail address.") ) license = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Text indicating the license covering the distribution') + required=False, + allow_blank=True, + help_text=_("Text indicating the license covering the distribution"), ) requires_python = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The Python version(s) that the distribution is guaranteed to be ' - 'compatible with.') + required=False, + allow_blank=True, + help_text=_( + "The Python version(s) that the distribution is guaranteed to be compatible with." + ), ) project_url = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A browsable URL for the project and a label for it, separated by a comma.') + required=False, + allow_blank=True, + help_text=_("A browsable URL for the project and a label for it, separated by a comma."), ) project_urls = serializers.JSONField( - required=False, default=dict, - help_text=_('A dictionary of labels and URLs for the project.') + required=False, + default=dict, + help_text=_("A dictionary of labels and URLs for the project."), ) platform = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A comma-separated list of platform specifications, ' - 'summarizing the operating systems supported by the package.') + required=False, + allow_blank=True, + help_text=_( + "A comma-separated list of platform specifications, " + "summarizing the operating systems supported by the package." + ), ) supported_platform = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Field to specify the OS and CPU for which the binary package was compiled. ') + required=False, + allow_blank=True, + help_text=_("Field to specify the OS and CPU for which the binary package was compiled. "), ) requires_dist = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing names of some other distutils project ' - 'required by this distribution.') + required=False, + default=list, + help_text=_( + "A JSON list containing names of some other distutils project " + "required by this distribution." + ), ) provides_dist = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing names of a Distutils project which is contained' - ' within this distribution.') + required=False, + default=list, + help_text=_( + "A JSON list containing names of a Distutils project which is contained" + " within this distribution." + ), ) obsoletes_dist = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing names of a distutils project\'s distribution which ' - 'this distribution renders obsolete, meaning that the two projects should not ' - 'be installed at the same time.') + required=False, + default=list, + help_text=_( + "A JSON list containing names of a distutils project's distribution which " + "this distribution renders obsolete, meaning that the two projects should not " + "be installed at the same time." + ), ) requires_external = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing some dependency in the system that the distribution ' - 'is to be used.') + required=False, + default=list, + help_text=_( + "A JSON list containing some dependency in the system that the distribution " + "is to be used." + ), ) classifiers = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing classification values for a Python package.') + required=False, + default=list, + help_text=_("A JSON list containing classification values for a Python package."), ) def deferred_validate(self, data): @@ -212,22 +250,26 @@ def deferred_validate(self, data): try: filename = data["relative_path"] except KeyError: - raise serializers.ValidationError(detail={"relative_path": _('This field is required')}) + raise serializers.ValidationError(detail={"relative_path": _("This field is required")}) artifact = data["artifact"] try: _data = artifact_to_python_content_data(filename, artifact, domain=get_domain()) except ValueError: - raise serializers.ValidationError(_( - "Extension on {} is not a valid python extension " - "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(filename) + raise serializers.ValidationError( + _( + "Extension on {} is not a valid python extension " + "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)" + ).format(filename) ) if data.get("sha256") and data["sha256"] != artifact.sha256: raise serializers.ValidationError( - detail={"sha256": _( - "The uploaded artifact's sha256 checksum does not match the one provided" - )} + detail={ + "sha256": _( + "The uploaded artifact's sha256 checksum does not match the one provided" + ) + } ) data.update(_data) @@ -242,11 +284,33 @@ def retrieve(self, validated_data): class Meta: fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + ( - 'filename', 'packagetype', 'name', 'version', 'sha256', 'metadata_version', 'summary', - 'description', 'description_content_type', 'keywords', 'home_page', 'download_url', - 'author', 'author_email', 'maintainer', 'maintainer_email', 'license', - 'requires_python', 'project_url', 'project_urls', 'platform', 'supported_platform', - 'requires_dist', 'provides_dist', 'obsoletes_dist', 'requires_external', 'classifiers' + "filename", + "packagetype", + "name", + "version", + "sha256", + "metadata_version", + "summary", + "description", + "description_content_type", + "keywords", + "home_page", + "download_url", + "author", + "author_email", + "maintainer", + "maintainer_email", + "license", + "requires_python", + "project_url", + "project_urls", + "platform", + "supported_platform", + "requires_dist", + "provides_dist", + "obsoletes_dist", + "requires_external", + "classifiers", ) model = python_models.PythonPackageContent @@ -258,7 +322,11 @@ class MinimalPythonPackageContentSerializer(PythonPackageContentSerializer): class Meta: fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + ( - 'filename', 'packagetype', 'name', 'version', 'sha256', + "filename", + "packagetype", + "name", + "version", + "sha256", ) model = python_models.PythonPackageContent @@ -282,47 +350,49 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer): child=serializers.CharField(allow_blank=False), required=False, allow_empty=True, - help_text=_( - "A list containing project specifiers for Python packages to include." - ), + help_text=_("A list containing project specifiers for Python packages to include."), ) excludes = serializers.ListField( child=serializers.CharField(allow_blank=False), required=False, allow_empty=True, - help_text=_( - "A list containing project specifiers for Python packages to exclude." - ), + help_text=_("A list containing project specifiers for Python packages to exclude."), ) prereleases = serializers.BooleanField( - required=False, - help_text=_('Whether or not to include pre-release packages in the sync.') + required=False, help_text=_("Whether or not to include pre-release packages in the sync.") ) policy = serializers.ChoiceField( - help_text=_("The policy to use when downloading content. The possible values include: " - "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default."), + help_text=_( + "The policy to use when downloading content. The possible values include: " + "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default." + ), choices=core_models.Remote.POLICY_CHOICES, - default=core_models.Remote.ON_DEMAND + default=core_models.Remote.ON_DEMAND, ) package_types = MultipleChoiceArrayField( required=False, - help_text=_("The package types to sync for Python content. Leave blank to get every" - "package type."), + help_text=_( + "The package types to sync for Python content. Leave blank to get everypackage type." + ), choices=python_models.PACKAGE_TYPES, - default=list + default=list, ) keep_latest_packages = serializers.IntegerField( required=False, - help_text=_("The amount of latest versions of a package to keep on sync, includes" - "pre-releases if synced. Default 0 keeps all versions."), - default=0 + help_text=_( + "The amount of latest versions of a package to keep on sync, includes" + "pre-releases if synced. Default 0 keeps all versions." + ), + default=0, ) exclude_platforms = MultipleChoiceArrayField( required=False, - help_text=_("List of platforms to exclude syncing Python packages for. Possible values" - "include: windows, macos, freebsd, and linux."), + help_text=_( + "List of platforms to exclude syncing Python packages for. Possible values" + "include: windows, macos, freebsd, and linux." + ), choices=python_models.PLATFORMS, - default=list + default=list, ) def validate_includes(self, value): @@ -332,7 +402,7 @@ def validate_includes(self, value): Requirement(pkg) except ValueError as ve: raise serializers.ValidationError( - _("includes specifier {} is invalid. {}".format(pkg, ve)) + _("includes specifier {} is invalid. {}").format(pkg, ve) ) return value @@ -343,13 +413,17 @@ def validate_excludes(self, value): Requirement(pkg) except ValueError as ve: raise serializers.ValidationError( - _("excludes specifier {} is invalid. {}".format(pkg, ve)) + _("excludes specifier {} is invalid. {}").format(pkg, ve) ) return value class Meta: fields = core_serializers.RemoteSerializer.Meta.fields + ( - "includes", "excludes", "prereleases", "package_types", "keep_latest_packages", + "includes", + "excludes", + "prereleases", + "package_types", + "keep_latest_packages", "exclude_platforms", ) model = python_models.PythonRemote @@ -371,10 +445,12 @@ class PythonBanderRemoteSerializer(serializers.Serializer): ) policy = serializers.ChoiceField( - help_text=_("The policy to use when downloading content. The possible values include: " - "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default."), + help_text=_( + "The policy to use when downloading content. The possible values include: " + "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default." + ), choices=core_models.Remote.POLICY_CHOICES, - default=core_models.Remote.ON_DEMAND + default=core_models.Remote.ON_DEMAND, ) @@ -384,8 +460,9 @@ class PythonPublicationSerializer(core_serializers.PublicationSerializer): """ distributions = core_serializers.DetailRelatedField( - help_text=_('This publication is currently being hosted as configured by these ' - 'distributions.'), + help_text=_( + "This publication is currently being hosted as configured by these distributions." + ), source="distribution_set", view_name="pythondistributions-detail", many=True, @@ -393,5 +470,5 @@ class PythonPublicationSerializer(core_serializers.PublicationSerializer): ) class Meta: - fields = core_serializers.PublicationSerializer.Meta.fields + ('distributions',) + fields = core_serializers.PublicationSerializer.Meta.fields + ("distributions",) model = python_models.PythonPublication diff --git a/pulp_python/app/settings.py b/pulp_python/app/settings.py index 7084ef109..eab951a50 100644 --- a/pulp_python/app/settings.py +++ b/pulp_python/app/settings.py @@ -1,7 +1,7 @@ import socket PYTHON_GROUP_UPLOADS = False -PYPI_API_HOSTNAME = 'https://' + socket.getfqdn() +PYPI_API_HOSTNAME = "https://" + socket.getfqdn() DRF_ACCESS_POLICY = { "dynaconf_merge_unique": True, diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index 136b511f6..ba0cb15b9 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -1,6 +1,6 @@ -from gettext import gettext as _ import logging import os +from gettext import gettext as _ from django.core.files import File from packaging.utils import canonicalize_name @@ -9,8 +9,7 @@ from pulpcore.plugin.util import get_domain from pulp_python.app import models as python_models -from pulp_python.app.utils import write_simple_index, write_simple_detail - +from pulp_python.app.utils import write_simple_detail, write_simple_index log = logging.getLogger(__name__) @@ -25,15 +24,17 @@ def publish(repository_version_pk): """ repository_version = models.RepositoryVersion.objects.get(pk=repository_version_pk) - log.info(_('Publishing: repository={repo}, version={version}').format( - repo=repository_version.repository.name, - version=repository_version.number, - )) + log.info( + _("Publishing: repository={repo}, version={version}").format( + repo=repository_version.repository.name, + version=repository_version.number, + ) + ) with python_models.PythonPublication.create(repository_version, pass_through=True) as pub: write_simple_api(pub) - log.info(_('Publication: {pk} created').format(pk=pub.pk)) + log.info(_("Publication: {pk} created").format(pk=pub.pk)) return pub @@ -51,26 +52,24 @@ def write_simple_api(publication): """ domain = get_domain() - simple_dir = 'simple/' + simple_dir = "simple/" os.mkdir(simple_dir) project_names = ( python_models.PythonPackageContent.objects.filter( pk__in=publication.repository_version.content, _pulp_domain=domain ) - .order_by('name') - .values_list('name', flat=True) + .order_by("name") + .values_list("name", flat=True) .distinct() ) # write the root index, which lists all of the projects for which there is a package available - index_path = '{simple_dir}index.html'.format(simple_dir=simple_dir) - with open(index_path, 'w') as index: + index_path = "{simple_dir}index.html".format(simple_dir=simple_dir) + with open(index_path, "w") as index: index.write(write_simple_index(project_names)) index_metadata = models.PublishedMetadata.create_from_file( - relative_path=index_path, - publication=publication, - file=File(open(index_path, 'rb')) + relative_path=index_path, publication=publication, file=File(open(index_path, "rb")) ) index_metadata.save() @@ -86,41 +85,41 @@ def write_simple_api(publication): current_name = project_names[ind] package_releases = [] for release in releases.iterator(): - if release['name'] != current_name: + if release["name"] != current_name: write_project_page( name=canonicalize_name(current_name), simple_dir=simple_dir, package_releases=package_releases, - publication=publication + publication=publication, ) package_releases = [] ind += 1 current_name = project_names[ind] - relative_path = release['filename'] + relative_path = release["filename"] path = f"../../{relative_path}" - checksum = release['sha256'] + checksum = release["sha256"] package_releases.append((relative_path, path, checksum)) # Write the final project's page write_project_page( name=canonicalize_name(current_name), simple_dir=simple_dir, package_releases=package_releases, - publication=publication + publication=publication, ) def write_project_page(name, simple_dir, package_releases, publication): """Writes a project's simple page.""" - project_dir = f'{simple_dir}{name}/' + project_dir = f"{simple_dir}{name}/" os.mkdir(project_dir) - metadata_relative_path = f'{project_dir}index.html' + metadata_relative_path = f"{project_dir}index.html" - with open(metadata_relative_path, 'w') as simple_metadata: + with open(metadata_relative_path, "w") as simple_metadata: simple_metadata.write(write_simple_detail(name, package_releases)) project_metadata = models.PublishedMetadata.create_from_file( relative_path=metadata_relative_path, publication=publication, - file=File(open(metadata_relative_path, 'rb')) + file=File(open(metadata_relative_path, "rb")), ) project_metadata.save() # change to bulk create when multi-table supported diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index 317f7b6bc..f5a7424a0 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -1,10 +1,15 @@ import logging - -from aiohttp import ClientResponseError, ClientError -from lxml.etree import LxmlError -from gettext import gettext as _ from functools import partial +from gettext import gettext as _ +from urllib.parse import urljoin +from aiohttp import ClientError, ClientResponseError +from bandersnatch.configuration import BandersnatchConfig +from bandersnatch.master import Master +from bandersnatch.mirror import Mirror +from lxml.etree import LxmlError +from packaging.requirements import Requirement +from pypi_simple import IndexPage from rest_framework import serializers from pulpcore.plugin.download import HttpDownloader @@ -20,14 +25,7 @@ PythonPackageContent, PythonRemote, ) -from pulp_python.app.utils import parse_metadata, PYPI_LAST_SERIAL -from pypi_simple import IndexPage - -from bandersnatch.mirror import Mirror -from bandersnatch.master import Master -from bandersnatch.configuration import BandersnatchConfig -from packaging.requirements import Requirement -from urllib.parse import urljoin +from pulp_python.app.utils import PYPI_LAST_SERIAL, parse_metadata logger = logging.getLogger(__name__) @@ -51,9 +49,7 @@ def sync(remote_pk, repository_pk, mirror): repository = Repository.objects.get(pk=repository_pk) if not remote.url: - raise serializers.ValidationError( - detail=_("A remote must have a url attribute to sync.") - ) + raise serializers.ValidationError(detail=_("A remote must have a url attribute to sync.")) first_stage = PythonBanderStage(remote) DeclarativeVersion(first_stage, repository, mirror).create() @@ -146,9 +142,7 @@ async def run(self): ) packages_to_sync = None if self.remote.includes: - packages_to_sync = [ - Requirement(pkg).name for pkg in self.remote.includes - ] + packages_to_sync = [Requirement(pkg).name for pkg in self.remote.includes] await pmirror.synchronize(packages_to_sync) # place back old session so that it is properly closed master.session = old_session @@ -159,9 +153,7 @@ class PulpMirror(Mirror): Pulp Mirror Class to perform syncing using Bandersnatch """ - def __init__( - self, serial, master, workers, deferred_download, python_stage, progress_report - ): + def __init__(self, serial, master, workers, deferred_download, python_stage, progress_report): """Initialize Bandersnatch Mirror""" super().__init__(master=master, workers=workers) self.synced_serial = serial @@ -176,11 +168,7 @@ async def determine_packages_to_sync(self): """ number_xmlrpc_attempts = 3 for attempt in range(number_xmlrpc_attempts): - logger.info( - "Attempt {} to get package list from {}".format( - attempt, self.master.url - ) - ) + logger.info("Attempt {} to get package list from {}".format(attempt, self.master.url)) try: if not self.synced_serial: logger.info("Syncing all packages.") @@ -192,9 +180,7 @@ async def determine_packages_to_sync(self): ) else: logger.info("Syncing based on changelog.") - changed_packages = await self.master.changed_packages( - self.synced_serial - ) + changed_packages = await self.master.changed_packages(self.synced_serial) self.packages_to_sync.update(changed_packages) self.target_serial = max( [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()] diff --git a/pulp_python/app/tasks/upload.py b/pulp_python/app/tasks/upload.py index d573c56ac..7f88e8d6a 100644 --- a/pulp_python/app/tasks/upload.py +++ b/pulp_python/app/tasks/upload.py @@ -1,9 +1,10 @@ import time - from datetime import datetime, timezone -from django.db import transaction + from django.contrib.sessions.models import Session -from pulpcore.plugin.models import Artifact, CreatedResource, ContentArtifact +from django.db import transaction + +from pulpcore.plugin.models import Artifact, ContentArtifact, CreatedResource from pulpcore.plugin.util import get_domain from pulp_python.app.models import PythonPackageContent, PythonRepository @@ -43,10 +44,10 @@ def upload_group(session_pk, repository_pk=None): with transaction.atomic(): session_data = s_query.first().get_decoded() now = datetime.now(tz=timezone.utc) - start_time = datetime.fromisoformat(session_data['start']) + start_time = datetime.fromisoformat(session_data["start"]) if now >= start_time: content_to_add = PythonPackageContent.objects.none() - for artifact_sha256, filename in session_data['artifacts']: + for artifact_sha256, filename in session_data["artifacts"]: pre_check = PythonPackageContent.objects.filter( sha256=artifact_sha256, _pulp_domain=domain ) @@ -81,9 +82,7 @@ def create_content(artifact_sha256, filename, domain): @transaction.atomic() def create(): content = PythonPackageContent.objects.create(**data) - ContentArtifact.objects.create( - artifact=artifact, content=content, relative_path=filename - ) + ContentArtifact.objects.create(artifact=artifact, content=content, relative_path=filename) return content new_content = create() diff --git a/pulp_python/app/urls.py b/pulp_python/app/urls.py index 0a7863334..f8352acc8 100644 --- a/pulp_python/app/urls.py +++ b/pulp_python/app/urls.py @@ -1,7 +1,7 @@ from django.conf import settings from django.urls import path -from pulp_python.app.pypi.views import SimpleView, MetadataView, PyPIView, UploadView +from pulp_python.app.pypi.views import MetadataView, PyPIView, SimpleView, UploadView if settings.DOMAIN_ENABLED: PYPI_API_URL = "pypi///" @@ -16,17 +16,17 @@ path( PYPI_API_URL + "pypi//", MetadataView.as_view({"get": "retrieve"}), - name="pypi-metadata" + name="pypi-metadata", ), path( PYPI_API_URL + "simple//", SimpleView.as_view({"get": "retrieve"}), - name="simple-package-detail" + name="simple-package-detail", ), path( - PYPI_API_URL + 'simple/', + PYPI_API_URL + "simple/", SimpleView.as_view({"get": "list", "post": "create"}), - name="simple-detail" + name="simple-detail", ), path(PYPI_API_URL, PyPIView.as_view({"get": "retrieve"}), name="pypi-detail"), ] diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index c3d350fa4..b49cca016 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -1,15 +1,15 @@ -import pkginfo +import json import re import shutil import tempfile -import json from collections import defaultdict + +import pkginfo from django.conf import settings from jinja2 import Template from packaging.utils import canonicalize_name from packaging.version import parse - PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" """TODO This serial constant is temporary until Python repositories implements serials""" PYPI_SERIAL_CONSTANT = 1000000000 @@ -58,10 +58,12 @@ r"""^(?P.+?)-(?P.*?) ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) \.whl|\.dist-info)$""", - re.VERBOSE + re.VERBOSE, ), # regex based on https://setuptools.pypa.io/en/latest/deprecated/python_eggs.html#filename-embedded-metadata # noqa: E501 - ".egg": re.compile(r"^(?P.+?)-(?P.*?)(-(?P.+?(-(?P.+?))?))?\.egg|\.egg-info$"), # noqa: E501 + ".egg": re.compile( + r"^(?P.+?)-(?P.*?)(-(?P.+?(-(?P.+?))?))?\.egg|\.egg-info$" + ), # noqa: E501 # regex based on https://github.com/python/cpython/blob/v3.7.0/Lib/distutils/command/bdist_wininst.py#L292 # noqa: E501 ".exe": re.compile(r"^(?P.+?)-(?P.*?)\.(?P.+?)(-(?P.+?))?\.exe$"), } @@ -86,32 +88,32 @@ def parse_project_metadata(project): """ package = {} - package['name'] = project.get('name') or "" - package['version'] = project.get('version') or "" - package['packagetype'] = project.get('packagetype') or "" - package['metadata_version'] = project.get('metadata_version') or "" - package['summary'] = project.get('summary') or "" - package['description'] = project.get('description') or "" - package['keywords'] = project.get('keywords') or "" - package['home_page'] = project.get('home_page') or "" - package['download_url'] = project.get('download_url') or "" - package['author'] = project.get('author') or "" - package['author_email'] = project.get('author_email') or "" - package['maintainer'] = project.get('maintainer') or "" - package['maintainer_email'] = project.get('maintainer_email') or "" - package['license'] = project.get('license') or "" - package['project_url'] = project.get('project_url') or "" - package['platform'] = project.get('platform') or "" - package['supported_platform'] = project.get('supported_platform') or "" - package['requires_python'] = project.get('requires_python') or "" - package['requires_dist'] = json.dumps(project.get('requires_dist', [])) - package['provides_dist'] = json.dumps(project.get('provides_dist', [])) - package['obsoletes_dist'] = json.dumps(project.get('obsoletes_dist', [])) - package['requires_external'] = json.dumps(project.get('requires_external', [])) - package['classifiers'] = json.dumps(project.get('classifiers', [])) - package['project_urls'] = json.dumps(project.get('project_urls', {})) - package['description_content_type'] = project.get('description_content_type') or "" - package['python_version'] = project.get('python_version') or "" + package["name"] = project.get("name") or "" + package["version"] = project.get("version") or "" + package["packagetype"] = project.get("packagetype") or "" + package["metadata_version"] = project.get("metadata_version") or "" + package["summary"] = project.get("summary") or "" + package["description"] = project.get("description") or "" + package["keywords"] = project.get("keywords") or "" + package["home_page"] = project.get("home_page") or "" + package["download_url"] = project.get("download_url") or "" + package["author"] = project.get("author") or "" + package["author_email"] = project.get("author_email") or "" + package["maintainer"] = project.get("maintainer") or "" + package["maintainer_email"] = project.get("maintainer_email") or "" + package["license"] = project.get("license") or "" + package["project_url"] = project.get("project_url") or "" + package["platform"] = project.get("platform") or "" + package["supported_platform"] = project.get("supported_platform") or "" + package["requires_python"] = project.get("requires_python") or "" + package["requires_dist"] = json.dumps(project.get("requires_dist", [])) + package["provides_dist"] = json.dumps(project.get("provides_dist", [])) + package["obsoletes_dist"] = json.dumps(project.get("obsoletes_dist", [])) + package["requires_external"] = json.dumps(project.get("requires_external", [])) + package["classifiers"] = json.dumps(project.get("classifiers", [])) + package["project_urls"] = json.dumps(project.get("project_urls", {})) + package["description_content_type"] = project.get("description_content_type") or "" + package["python_version"] = project.get("python_version") or "" return package @@ -134,13 +136,15 @@ def parse_metadata(project, version, distribution): """ package = parse_project_metadata(project) - package['filename'] = distribution.get('filename') or "" - package['packagetype'] = distribution.get('packagetype') or "" - package['version'] = version - package['url'] = distribution.get('url') or "" - package['sha256'] = distribution.get('digests', {}).get('sha256') or "" - package['python_version'] = distribution.get('python_version') or package.get('python_version') - package['requires_python'] = distribution.get('requires_python') or package.get('requires_python') # noqa: E501 + package["filename"] = distribution.get("filename") or "" + package["packagetype"] = distribution.get("packagetype") or "" + package["version"] = version + package["url"] = distribution.get("url") or "" + package["sha256"] = distribution.get("digests", {}).get("sha256") or "" + package["python_version"] = distribution.get("python_version") or package.get("python_version") + package["requires_python"] = distribution.get("requires_python") or package.get( + "requires_python" + ) # noqa: E501 return package @@ -159,7 +163,7 @@ def get_project_metadata_from_artifact(filename, artifact): # Copy file to a temp directory under the user provided filename, we do this # because pkginfo validates that the filename has a valid extension before # reading it - with tempfile.NamedTemporaryFile('wb', dir=".", suffix=filename) as temp_file: + with tempfile.NamedTemporaryFile("wb", dir=".", suffix=filename) as temp_file: shutil.copyfileobj(artifact.file, temp_file) temp_file.flush() metadata = DIST_TYPES[packagetype](temp_file.name) @@ -181,10 +185,10 @@ def artifact_to_python_content_data(filename, artifact, domain=None): """ metadata = get_project_metadata_from_artifact(filename, artifact) data = parse_project_metadata(vars(metadata)) - data['sha256'] = artifact.sha256 - data['filename'] = filename - data['pulp_domain'] = domain or artifact.pulp_domain - data['_pulp_domain'] = data['pulp_domain'] + data["sha256"] = artifact.sha256 + data["filename"] = filename + data["pulp_domain"] = domain or artifact.pulp_domain + data["_pulp_domain"] = data["pulp_domain"] return data @@ -306,10 +310,12 @@ def python_content_to_download_info(content, base_path, domain=None): Takes in a PythonPackageContent and base path of the distribution to create a dictionary of download information for that content. This dictionary is used by Releases and Urls. """ + def find_artifact(): _art = content_artifact.artifact if not _art: from pulpcore.plugin import models + _art = models.RemoteArtifact.objects.filter(content_artifact=content_artifact).first() return _art @@ -340,7 +346,7 @@ def find_artifact(): "upload_time_iso_8601": str(content.pulp_created.isoformat()), "url": url, "yanked": False, - "yanked_reason": None + "yanked_reason": None, } diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 596e76fbc..ddc39215c 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -26,7 +26,7 @@ class PythonRepositoryViewSet( synced, added, or removed. """ - endpoint_name = 'python' + endpoint_name = "python" queryset = python_models.PythonRepository.objects.all() serializer_class = python_serializers.PythonRepositorySerializer queryset_filtering_required_permission = "python.view_pythonrepository" @@ -122,11 +122,8 @@ class PythonRepositoryViewSet( "python.pythonrepository_viewer": ["python.view_pythonrepository"], } - @extend_schema( - summary="Sync from remote", - responses={202: AsyncOperationResponseSerializer} - ) - @action(detail=True, methods=['post'], serializer_class=RepositorySyncURLSerializer) + @extend_schema(summary="Sync from remote", responses={202: AsyncOperationResponseSerializer}) + @action(detail=True, methods=["post"], serializer_class=RepositorySyncURLSerializer) def sync(self, request, pk): """ @@ -136,22 +133,21 @@ def sync(self, request, pk): """ repository = self.get_object() serializer = RepositorySyncURLSerializer( - data=request.data, - context={'request': request, "repository_pk": pk} + data=request.data, context={"request": request, "repository_pk": pk} ) serializer.is_valid(raise_exception=True) - remote = serializer.validated_data.get('remote', repository.remote) - mirror = serializer.validated_data.get('mirror') + remote = serializer.validated_data.get("remote", repository.remote) + mirror = serializer.validated_data.get("mirror") result = dispatch( tasks.sync, exclusive_resources=[repository], shared_resources=[remote], kwargs={ - 'remote_pk': str(remote.pk), - 'repository_pk': str(repository.pk), - 'mirror': mirror - } + "remote_pk": str(remote.pk), + "repository_pk": str(repository.pk), + "mirror": mirror, + }, ) return core_viewsets.OperationPostponedResponse(result, request) @@ -169,8 +165,7 @@ class PythonRepositoryVersionViewSet(core_viewsets.RepositoryVersionViewSet): "action": ["list", "retrieve"], "principal": "authenticated", "effect": "allow", - "condition": - "has_repository_model_or_domain_or_obj_perms:python.view_pythonrepository", + "condition": "has_repository_model_or_domain_or_obj_perms:python.view_pythonrepository", }, { "action": ["destroy"], @@ -205,7 +200,7 @@ class PythonDistributionViewSet(core_viewsets.DistributionViewSet, core_viewsets href="./#tag/Content:-Packages">Python Package Content. """ - endpoint_name = 'pypi' + endpoint_name = "pypi" queryset = python_models.PythonDistribution.objects.all() serializer_class = python_serializers.PythonDistributionSerializer queryset_filtering_required_permission = "python.view_pythondistribution" @@ -294,19 +289,20 @@ class PythonPackageContentFilter(core_viewsets.ContentFilter): class Meta: model = python_models.PythonPackageContent fields = { - 'name': ['exact', 'in'], - 'author': ['exact', 'in'], - 'packagetype': ['exact', 'in'], - 'requires_python': ['exact', 'in', "contains"], - 'filename': ['exact', 'in', 'contains'], - 'keywords': ['in', 'contains'], - 'sha256': ['exact', 'in'], - 'version': ['exact', 'gt', 'lt', 'gte', 'lte'] + "name": ["exact", "in"], + "author": ["exact", "in"], + "packagetype": ["exact", "in"], + "requires_python": ["exact", "in", "contains"], + "filename": ["exact", "in", "contains"], + "keywords": ["in", "contains"], + "sha256": ["exact", "in"], + "version": ["exact", "gt", "lt", "gte", "lte"], } class PythonPackageSingleArtifactContentUploadViewSet( - core_viewsets.SingleArtifactContentUploadViewSet): + core_viewsets.SingleArtifactContentUploadViewSet +): """ PythonPackageContent represents each individually installable Python package. In the Python @@ -317,7 +313,7 @@ class PythonPackageSingleArtifactContentUploadViewSet( """ - endpoint_name = 'packages' + endpoint_name = "packages" queryset = python_models.PythonPackageContent.objects.all() serializer_class = python_serializers.PythonPackageContentSerializer minimal_serializer_class = python_serializers.MinimalPythonPackageContentSerializer @@ -354,7 +350,7 @@ class PythonRemoteViewSet(core_viewsets.RemoteViewSet, core_viewsets.RolesMixin) """ - endpoint_name = 'python' + endpoint_name = "python" queryset = python_models.PythonRemote.objects.all() serializer_class = python_serializers.PythonRemoteSerializer queryset_filtering_required_permission = "python.view_pythonremote" @@ -426,8 +422,11 @@ class PythonRemoteViewSet(core_viewsets.RemoteViewSet, core_viewsets.RolesMixin) summary="Create from Bandersnatch", responses={201: python_serializers.PythonRemoteSerializer}, ) - @action(detail=False, methods=["post"], - serializer_class=python_serializers.PythonBanderRemoteSerializer) + @action( + detail=False, + methods=["post"], + serializer_class=python_serializers.PythonBanderRemoteSerializer, + ) def from_bandersnatch(self, request): """ @@ -439,11 +438,12 @@ def from_bandersnatch(self, request): name = serializer.validated_data.get("name") policy = serializer.validated_data.get("policy") bander_config = BandersnatchConfig(bander_config_file.file.name).config - data = {"name": name, - "policy": policy, - "url": bander_config.get("mirror", "master"), - "download_concurrency": bander_config.get("mirror", "workers"), - } + data = { + "name": name, + "policy": policy, + "url": bander_config.get("mirror", "master"), + "download_concurrency": bander_config.get("mirror", "workers"), + } enabled = bander_config.get("plugins", "enabled") enabled_all = "all" in enabled data["prereleases"] = not (enabled_all or "prerelease_release" in enabled) @@ -460,8 +460,9 @@ def from_bandersnatch(self, request): "exclude_platform": ("blocklist", "platforms", "exclude_platforms"), } for plugin, options in plugin_filters.items(): - if (enabled_all or plugin in enabled) and \ - bander_config.has_option(options[0], options[1]): + if (enabled_all or plugin in enabled) and bander_config.has_option( + options[0], options[1] + ): data[options[2]] = bander_config.get(options[0], options[1]).split() remote = python_serializers.PythonRemoteSerializer(data=data, context={"request": request}) remote.is_valid(raise_exception=True) @@ -478,7 +479,7 @@ class PythonPublicationViewSet(core_viewsets.PublicationViewSet, core_viewsets.R """ - endpoint_name = 'pypi' + endpoint_name = "pypi" queryset = python_models.PythonPublication.objects.exclude(complete=False) serializer_class = python_serializers.PythonPublicationSerializer queryset_filtering_required_permission = "python.view_pythonpublication" @@ -542,9 +543,7 @@ class PythonPublicationViewSet(core_viewsets.PublicationViewSet, core_viewsets.R "python.pythonpublication_viewer": ["python.view_pythonpublication"], } - @extend_schema( - responses={202: AsyncOperationResponseSerializer} - ) + @extend_schema(responses={202: AsyncOperationResponseSerializer}) def create(self, request): """ @@ -552,18 +551,16 @@ def create(self, request): """ serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) - repository_version = serializer.validated_data.get('repository_version') + repository_version = serializer.validated_data.get("repository_version") # Safe because version OR repository is enforced by serializer. if not repository_version: - repository = serializer.validated_data.get('repository') + repository = serializer.validated_data.get("repository") repository_version = RepositoryVersion.latest(repository) result = dispatch( tasks.publish, shared_resources=[repository_version.repository], - kwargs={ - 'repository_version_pk': str(repository_version.pk) - } + kwargs={"repository_version_pk": str(repository_version.pk)}, ) return core_viewsets.OperationPostponedResponse(result, request) diff --git a/pulp_python/pytest_plugin.py b/pulp_python/pytest_plugin.py index 346cc9515..6c79723a2 100644 --- a/pulp_python/pytest_plugin.py +++ b/pulp_python/pytest_plugin.py @@ -1,16 +1,17 @@ -import pytest -import uuid import subprocess +import uuid + +import pytest from pulpcore.tests.functional.utils import BindingsNamespace + from pulp_python.tests.functional.constants import ( - PYTHON_FIXTURE_URL, - PYTHON_XS_PROJECT_SPECIFIER, PYTHON_EGG_FILENAME, + PYTHON_FIXTURE_URL, PYTHON_URL, + PYTHON_XS_PROJECT_SPECIFIER, ) - # Bindings API Fixtures @@ -35,6 +36,7 @@ def python_bindings(_api_client_set, bindings_cfg): @pytest.fixture def python_repo_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Repository with auto-cleanup.""" + def _gen_python_repo(remote=None, pulp_domain=None, **body): body.setdefault("name", str(uuid.uuid4())) kwargs = {} @@ -56,6 +58,7 @@ def python_repo(python_repo_factory): @pytest.fixture def python_distribution_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Distribution with auto-cleanup.""" + def _gen_python_distribution( publication=None, repository=None, version=None, pulp_domain=None, **body ): @@ -85,6 +88,7 @@ def _gen_python_distribution( @pytest.fixture def python_publication_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Publication with auto-cleanup.""" + def _gen_python_publication(repository, version=None, pulp_domain=None): repo_href = get_href(repository) if version: @@ -106,6 +110,7 @@ def _gen_python_publication(repository, version=None, pulp_domain=None): @pytest.fixture def python_remote_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Remote with auto-cleanup.""" + def _gen_python_remote(url=PYTHON_FIXTURE_URL, includes=None, pulp_domain=None, **body): body.setdefault("name", str(uuid.uuid4())) body.setdefault("url", url) @@ -125,6 +130,7 @@ def python_repo_with_sync( python_bindings, python_repo_factory, python_remote_factory, monitor_task ): """A factory to generate a Python Repository synced with the passed in Remote.""" + def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body): kwargs = {} if pulp_domain := body.get("pulp_domain"): @@ -141,6 +147,7 @@ def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body): @pytest.fixture def download_python_file(tmp_path, http_get): """Download a Python file and return its path.""" + def _download_python_file(relative_path, url): file_path = tmp_path / relative_path with open(file_path, mode="wb") as f: @@ -159,6 +166,7 @@ def python_file(download_python_file): @pytest.fixture def python_content_factory(python_bindings, download_python_file, monitor_task): """A factory to create a Python Package Content.""" + def _gen_python_content(relative_path=PYTHON_EGG_FILENAME, url=None, **body): body["relative_path"] = relative_path if url: @@ -191,6 +199,7 @@ def shelf_reader_cleanup(): @pytest.fixture def python_content_summary(python_bindings): """Get a summary of the repository version's content.""" + def _gen_summary(repository_version=None, repository=None, version=None): if repository_version is None: repo_href = get_href(repository) diff --git a/pulp_python/tests/functional/api/test_consume_content.py b/pulp_python/tests/functional/api/test_consume_content.py index 98ac094ee..56ee21400 100644 --- a/pulp_python/tests/functional/api/test_consume_content.py +++ b/pulp_python/tests/functional/api/test_consume_content.py @@ -1,5 +1,4 @@ import subprocess - from urllib.parse import urlsplit diff --git a/pulp_python/tests/functional/api/test_crud_content_unit.py b/pulp_python/tests/functional/api/test_crud_content_unit.py index 4264d0512..2cdaf0b25 100644 --- a/pulp_python/tests/functional/api/test_crud_content_unit.py +++ b/pulp_python/tests/functional/api/test_crud_content_unit.py @@ -1,14 +1,15 @@ -import pytest - from urllib.parse import urljoin + +import pytest from pypi_simple import PyPISimple from pulpcore.tests.functional.utils import PulpTaskError + from pulp_python.tests.functional.constants import ( - PYTHON_FIXTURES_URL, - PYTHON_PACKAGE_DATA, PYTHON_EGG_FILENAME, PYTHON_EGG_URL, + PYTHON_FIXTURES_URL, + PYTHON_PACKAGE_DATA, PYTHON_SM_FIXTURE_CHECKSUMS, ) @@ -102,7 +103,9 @@ def test_content_crud( monitor_task(pulpcore_bindings.OrphansCleanupApi.cleanup({"orphan_protection_time": 0}).task) mismatch_sha256 = PYTHON_SM_FIXTURE_CHECKSUMS["aiohttp-3.3.0.tar.gz"] content_body = { - "relative_path": PYTHON_EGG_FILENAME, "file": python_file, "sha256": mismatch_sha256 + "relative_path": PYTHON_EGG_FILENAME, + "file": python_file, + "sha256": mismatch_sha256, } with pytest.raises(PulpTaskError) as e: response = python_bindings.ContentPackagesApi.create(**content_body) diff --git a/pulp_python/tests/functional/api/test_crud_publications.py b/pulp_python/tests/functional/api/test_crud_publications.py index 70e8b4322..3e75a7285 100644 --- a/pulp_python/tests/functional/api/test_crud_publications.py +++ b/pulp_python/tests/functional/api/test_crud_publications.py @@ -1,12 +1,13 @@ -import pytest import random from urllib.parse import urljoin +import pytest + from pulp_python.tests.functional.constants import ( - PYTHON_SM_PROJECT_SPECIFIER, - PYTHON_SM_FIXTURE_RELEASES, - PYTHON_SM_FIXTURE_CHECKSUMS, PYTHON_EGG_FILENAME, + PYTHON_SM_FIXTURE_CHECKSUMS, + PYTHON_SM_FIXTURE_RELEASES, + PYTHON_SM_PROJECT_SPECIFIER, PYTHON_WHEEL_FILENAME, ) from pulp_python.tests.functional.utils import ensure_simple @@ -17,6 +18,7 @@ def python_publication_workflow( python_repo_with_sync, python_remote_factory, python_publication_factory ): """Create repo, remote, sync & then publish.""" + def _publish_workflow(repository=None, remote=None, **remote_body): if not remote: remote = python_remote_factory(**remote_body) @@ -55,8 +57,9 @@ def test_all_content_published(python_publication_workflow, python_distribution_ distro = python_distribution_factory(publication=pub) url = urljoin(distro.base_url, "simple/") - proper, msgs = ensure_simple(url, PYTHON_SM_FIXTURE_RELEASES, - sha_digests=PYTHON_SM_FIXTURE_CHECKSUMS) + proper, msgs = ensure_simple( + url, PYTHON_SM_FIXTURE_RELEASES, sha_digests=PYTHON_SM_FIXTURE_CHECKSUMS + ) assert proper is True, msgs diff --git a/pulp_python/tests/functional/api/test_crud_remotes.py b/pulp_python/tests/functional/api/test_crud_remotes.py index ace928fd3..eaaec3659 100644 --- a/pulp_python/tests/functional/api/test_crud_remotes.py +++ b/pulp_python/tests/functional/api/test_crud_remotes.py @@ -1,11 +1,12 @@ -import pytest import uuid +import pytest + from pulp_python.tests.functional.constants import ( BANDERSNATCH_CONF, DEFAULT_BANDER_REMOTE_BODY, - PYTHON_INVALID_SPECIFIER_NO_NAME, PYTHON_INVALID_SPECIFIER_BAD_VERSION, + PYTHON_INVALID_SPECIFIER_NO_NAME, PYTHON_VALID_SPECIFIER_NO_VERSION, ) diff --git a/pulp_python/tests/functional/api/test_domains.py b/pulp_python/tests/functional/api/test_domains.py index 0f5233a44..13660d1a7 100644 --- a/pulp_python/tests/functional/api/test_domains.py +++ b/pulp_python/tests/functional/api/test_domains.py @@ -1,17 +1,17 @@ -import pytest -import uuid import json import subprocess +import uuid +from urllib.parse import urlsplit + +import pytest -from pulpcore.app import settings +from pulpcore.app import settings # noqa: TID251 from pulp_python.tests.functional.constants import ( PYTHON_EGG_FILENAME, - PYTHON_SM_PROJECT_SPECIFIER, PYTHON_SM_PACKAGE_COUNT, + PYTHON_SM_PROJECT_SPECIFIER, ) -from urllib.parse import urlsplit - pytestmark = pytest.mark.skipif(not settings.DOMAIN_ENABLED, reason="Domain not enabled") @@ -72,7 +72,9 @@ def test_domain_object_creation( with pytest.raises(python_bindings.ApiException) as e: distro_body = { - "name": str(uuid.uuid4()), "base_path": str(uuid.uuid4()), "repository": repo.pulp_href + "name": str(uuid.uuid4()), + "base_path": str(uuid.uuid4()), + "repository": repo.pulp_href, } python_bindings.DistributionsPypiApi.create(distro_body) assert e.value.status == 400 diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index 11ca94494..c9820f114 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -1,10 +1,10 @@ import pytest from pulp_python.tests.functional.constants import ( - PYTHON_MD_PROJECT_SPECIFIER, + PYTHON_LG_PACKAGE_COUNT, PYTHON_LG_PROJECT_SPECIFIER, PYTHON_MD_PACKAGE_COUNT, - PYTHON_LG_PACKAGE_COUNT, + PYTHON_MD_PROJECT_SPECIFIER, ) diff --git a/pulp_python/tests/functional/api/test_export_import.py b/pulp_python/tests/functional/api/test_export_import.py index 15a21e044..7bdd66f9d 100644 --- a/pulp_python/tests/functional/api/test_export_import.py +++ b/pulp_python/tests/functional/api/test_export_import.py @@ -4,15 +4,18 @@ NOTE: assumes ALLOWED_EXPORT_PATHS setting contains "/tmp" - all tests will fail if this is not the case. """ -import pytest + import uuid -from pulpcore.app import settings +import pytest + +from pulpcore.app import settings # noqa: TID251 + from pulp_python.tests.functional.constants import ( - PYTHON_XS_PROJECT_SPECIFIER, PYTHON_SM_PROJECT_SPECIFIER + PYTHON_SM_PROJECT_SPECIFIER, + PYTHON_XS_PROJECT_SPECIFIER, ) - pytestmark = [ pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domains do not support export."), pytest.mark.skipif( diff --git a/pulp_python/tests/functional/api/test_full_mirror.py b/pulp_python/tests/functional/api/test_full_mirror.py index 2f4380090..520e9ac47 100644 --- a/pulp_python/tests/functional/api/test_full_mirror.py +++ b/pulp_python/tests/functional/api/test_full_mirror.py @@ -1,15 +1,15 @@ +import subprocess +from urllib.parse import urljoin, urlsplit + import pytest import requests -import subprocess +from pypi_simple import ProjectPage from pulp_python.tests.functional.constants import ( PYPI_URL, PYTHON_XS_FIXTURE_CHECKSUMS, ) -from pypi_simple import ProjectPage -from urllib.parse import urljoin, urlsplit - def test_pull_through_install( python_bindings, python_remote_factory, python_distribution_factory, delete_orphans_pre diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index 034f8f83f..51ab06c9d 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -1,27 +1,25 @@ -import pytest -import requests import subprocess - from urllib.parse import urljoin +import pytest +import requests + from pulp_python.tests.functional.constants import ( - PYTHON_SM_PROJECT_SPECIFIER, - PYTHON_SM_FIXTURE_RELEASES, - PYTHON_SM_FIXTURE_CHECKSUMS, - PYTHON_MD_PROJECT_SPECIFIER, - PYTHON_MD_PYPI_SUMMARY, PYTHON_EGG_FILENAME, - PYTHON_EGG_URL, PYTHON_EGG_SHA256, + PYTHON_EGG_URL, + PYTHON_MD_PROJECT_SPECIFIER, + PYTHON_MD_PYPI_SUMMARY, + PYTHON_SM_FIXTURE_CHECKSUMS, + PYTHON_SM_FIXTURE_RELEASES, + PYTHON_SM_PROJECT_SPECIFIER, PYTHON_WHEEL_FILENAME, - PYTHON_WHEEL_URL, PYTHON_WHEEL_SHA256, + PYTHON_WHEEL_URL, SHELF_PYTHON_JSON, ) - from pulp_python.tests.functional.utils import ensure_simple - PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" PYPI_SERIAL_CONSTANT = 1000000000 @@ -29,6 +27,7 @@ @pytest.fixture def python_empty_repo_distro(python_repo_factory, python_distribution_factory): """Returns an empty repo with and distribution serving it.""" + def _generate_empty_repo_distro(repo_body=None, distro_body=None): repo_body = repo_body or {} distro_body = distro_body or {} @@ -302,9 +301,7 @@ def test_pypi_last_serial( repo = python_repo_with_sync(remote) pub = python_publication_factory(repository=repo) distro = python_distribution_factory(publication=pub) - content_url = urljoin( - pulp_content_url, f"{distro.base_path}/pypi/shelf-reader/json" - ) + content_url = urljoin(pulp_content_url, f"{distro.base_path}/pypi/shelf-reader/json") pypi_url = urljoin(distro.base_url, "pypi/shelf-reader/json/") for url in [content_url, pypi_url]: response = requests.get(url) @@ -317,9 +314,7 @@ def assert_pypi_json(package): assert SHELF_PYTHON_JSON["last_serial"] == package["last_serial"] assert SHELF_PYTHON_JSON["info"].items() <= package["info"].items() assert len(SHELF_PYTHON_JSON["urls"]) == len(package["urls"]) - assert_download_info( - SHELF_PYTHON_JSON["urls"], package["urls"], "Failed to match URLS" - ) + assert_download_info(SHELF_PYTHON_JSON["urls"], package["urls"], "Failed to match URLS") assert SHELF_PYTHON_JSON["releases"].keys() <= package["releases"].keys() for version in SHELF_PYTHON_JSON["releases"].keys(): assert_download_info( diff --git a/pulp_python/tests/functional/api/test_rbac.py b/pulp_python/tests/functional/api/test_rbac.py index 125ef368f..4990b1430 100644 --- a/pulp_python/tests/functional/api/test_rbac.py +++ b/pulp_python/tests/functional/api/test_rbac.py @@ -1,6 +1,7 @@ -import pytest import uuid +import pytest + from pulp_python.tests.functional.constants import ( PYTHON_EGG_FILENAME, PYTHON_EGG_SHA256, @@ -44,9 +45,9 @@ def _try_action(user, client, action, outcome, *args, **kwargs): except python_bindings.module.ApiException as e: assert e.status == outcome, f"{e}" else: - assert ( - status_code == outcome - ), f"User performed {action} when they shouldn't been able to" + assert status_code == outcome, ( + f"User performed {action} when they shouldn't been able to" + ) return data return _try_action @@ -238,7 +239,7 @@ def test_pypi_apis( python_distribution_factory, anonymous_user, download_python_file, - try_action + try_action, ): alice, bob, charlie = gen_users(["pythonrepository", "pythondistribution"]) with bob: diff --git a/pulp_python/tests/functional/api/test_repair.py b/pulp_python/tests/functional/api/test_repair.py index 792d49c28..005df5a20 100644 --- a/pulp_python/tests/functional/api/test_repair.py +++ b/pulp_python/tests/functional/api/test_repair.py @@ -1,6 +1,7 @@ -import pytest import subprocess +import pytest + from pulp_python.tests.functional.constants import PYTHON_EGG_FILENAME @@ -65,7 +66,7 @@ def test_metadata_repair_command( move_to_repository(python_repo.pulp_href, [content.pulp_href]) process = subprocess.run( ["pulpcore-manager", "repair-python-metadata", "--repositories", python_repo.pulp_href], - capture_output=True + capture_output=True, ) assert process.returncode == 0 output = process.stdout.decode().strip() diff --git a/pulp_python/tests/functional/api/test_sync.py b/pulp_python/tests/functional/api/test_sync.py index 0a846cf61..f20e83cfc 100644 --- a/pulp_python/tests/functional/api/test_sync.py +++ b/pulp_python/tests/functional/api/test_sync.py @@ -1,21 +1,21 @@ import pytest from pulp_python.tests.functional.constants import ( - PYTHON_XS_PACKAGE_COUNT, - PYTHON_PRERELEASE_TEST_SPECIFIER, - PYTHON_WITH_PRERELEASE_COUNT, - PYTHON_WITHOUT_PRERELEASE_COUNT, - PYTHON_XS_PROJECT_SPECIFIER, - PYTHON_MD_PROJECT_SPECIFIER, + DJANGO_LATEST_3, + PYTHON_LG_FIXTURE_COUNTS, + PYTHON_LG_PACKAGE_COUNT, + PYTHON_LG_PROJECT_SPECIFIER, PYTHON_MD_PACKAGE_COUNT, - PYTHON_SM_PROJECT_SPECIFIER, + PYTHON_MD_PROJECT_SPECIFIER, + PYTHON_PRERELEASE_TEST_SPECIFIER, PYTHON_SM_PACKAGE_COUNT, + PYTHON_SM_PROJECT_SPECIFIER, PYTHON_UNAVAILABLE_PACKAGE_COUNT, PYTHON_UNAVAILABLE_PROJECT_SPECIFIER, - PYTHON_LG_PROJECT_SPECIFIER, - PYTHON_LG_PACKAGE_COUNT, - PYTHON_LG_FIXTURE_COUNTS, - DJANGO_LATEST_3, + PYTHON_WITH_PRERELEASE_COUNT, + PYTHON_WITHOUT_PRERELEASE_COUNT, + PYTHON_XS_PACKAGE_COUNT, + PYTHON_XS_PROJECT_SPECIFIER, SCIPY_COUNTS, ) @@ -213,7 +213,7 @@ def test_sync_package_type(python_repo_with_sync, python_remote_factory, python_ @pytest.mark.parallel def test_sync_platform_exclude( - python_repo_with_sync, python_remote_factory, python_content_summary + python_repo_with_sync, python_remote_factory, python_content_summary ): """ Tests for platform specific packages not being synced when specified @@ -226,8 +226,11 @@ def test_sync_platform_exclude( 1 any platform release """ # Tests that no windows packages are synced - remote = python_remote_factory(includes=["scipy"], exclude_platforms=["windows"], - prereleases=True, ) + remote = python_remote_factory( + includes=["scipy"], + exclude_platforms=["windows"], + prereleases=True, + ) repo = python_repo_with_sync(remote) summary = python_content_summary(repository_version=repo.latest_version_href) @@ -235,8 +238,11 @@ def test_sync_platform_exclude( assert summary.present["python.python"]["count"] == diff_count # Tests that no macos packages are synced - remote = python_remote_factory(includes=["scipy"], exclude_platforms=["macos"], - prereleases=True, ) + remote = python_remote_factory( + includes=["scipy"], + exclude_platforms=["macos"], + prereleases=True, + ) repo = python_repo_with_sync(remote) summary = python_content_summary(repository_version=repo.latest_version_href) @@ -244,8 +250,11 @@ def test_sync_platform_exclude( assert summary.present["python.python"]["count"] == diff_count # Tests that no linux packages are synced - remote = python_remote_factory(includes=["scipy"], exclude_platforms=["linux"], - prereleases=True, ) + remote = python_remote_factory( + includes=["scipy"], + exclude_platforms=["linux"], + prereleases=True, + ) repo = python_repo_with_sync(remote) summary = python_content_summary(repository_version=repo.latest_version_href) @@ -273,7 +282,7 @@ def test_sync_multiple_filters( includes=PYTHON_LG_PROJECT_SPECIFIER, package_types=["bdist_wheel"], keep_latest_packages=1, - prereleases=False + prereleases=False, ) repo = python_repo_with_sync(remote) diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index 231ae1bcf..b61888f11 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -1,7 +1,6 @@ import os from urllib.parse import urljoin - PULP_FIXTURES_BASE_URL = os.environ.get( "REMOTE_FIXTURES_ORIGIN", "https://fixtures.pulpproject.org/" ) @@ -25,9 +24,7 @@ "pylint", # matches 0 ] PYTHON_UNAVAILABLE_PACKAGE_COUNT = 5 -PYTHON_UNAVAILABLE_FIXTURE_SUMMARY = { - PYTHON_CONTENT_NAME: PYTHON_UNAVAILABLE_PACKAGE_COUNT -} +PYTHON_UNAVAILABLE_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_UNAVAILABLE_PACKAGE_COUNT} # no "name" field PYTHON_INVALID_SPECIFIER_NO_NAME = [ @@ -51,13 +48,9 @@ "Django", ] PYTHON_WITH_PRERELEASE_COUNT = 46 -PYTHON_WITH_PRERELEASE_FIXTURE_SUMMARY = { - PYTHON_CONTENT_NAME: PYTHON_WITH_PRERELEASE_COUNT -} +PYTHON_WITH_PRERELEASE_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_WITH_PRERELEASE_COUNT} PYTHON_WITHOUT_PRERELEASE_COUNT = 30 -PYTHON_WITHOUT_PRERELEASE_FIXTURE_SUMMARY = { - PYTHON_CONTENT_NAME: PYTHON_WITHOUT_PRERELEASE_COUNT -} +PYTHON_WITHOUT_PRERELEASE_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_WITHOUT_PRERELEASE_COUNT} # Specifier for basic sync / publish tests. PYTHON_XS_PROJECT_SPECIFIER = ["shelf-reader"] # matches 2 @@ -93,21 +86,15 @@ "aiohttp-3.2.1.tar.gz": "1b95d53f8dac13898f0a3e4af76f6f36d540fbfaefc4f4c9f43e436fa0e53d22", "aiohttp-3.2.0.tar.gz": "1be3903fe6a36d20492e74efb326522dd4702bf32b45ffc7acbc0fb34ab240a6", "Django-1.10.4.tar.gz": "fff7f062e510d812badde7cfc57745b7779edb4d209b2bc5ea8d954c22305c2b", - "Django-1.10.4-py2.py3-none-any.whl": - "a8e1a552205cda15023c39ecf17f7e525e96c5b0142e7879e8bd0c445351f2cc", + "Django-1.10.4-py2.py3-none-any.whl": "a8e1a552205cda15023c39ecf17f7e525e96c5b0142e7879e8bd0c445351f2cc", "Django-1.10.3.tar.gz": "6f92f08dee8a1bd7680e098a91bf5acd08b5cdfe74137f695b60fd79f4478c30", - "Django-1.10.3-py2.py3-none-any.whl": - "94426cc28d8721fbf13c333053f08d32427671a4ca7986f7030fc82bdf9c88c1", + "Django-1.10.3-py2.py3-none-any.whl": "94426cc28d8721fbf13c333053f08d32427671a4ca7986f7030fc82bdf9c88c1", "Django-1.10.2.tar.gz": "e127f12a0bfb34843b6e8c82f91e26fff6445a7ca91d222c0794174cf97cbce1", - "Django-1.10.2-py2.py3-none-any.whl": - "4d48ab8e84a7c8b2bc4b2f4f199bc3a8bfcc9cbdbc29e355ac5c44a501d73a1a", + "Django-1.10.2-py2.py3-none-any.whl": "4d48ab8e84a7c8b2bc4b2f4f199bc3a8bfcc9cbdbc29e355ac5c44a501d73a1a", "Django-1.10.1.tar.gz": "d6e6c5b25cb67f46afd7c82f536529b11981183423dad8932e15bce93d1a24f3", - "Django-1.10.1-py2.py3-none-any.whl": - "3d689905cd0635bbb33b87f9a5df7ca70a3db206faae4ec58cda5e7f5f47050d", - "celery-4.2.0-py2.py3-none-any.whl": - "2082cbd82effa8ac8a8a58977d70bb203a9f362817e3b66f4578117b9f93d8a9", - "celery-4.1.1-py2.py3-none-any.whl": - "6fc4678d1692af97e137b2a9f1c04efd8e7e2fb7134c5c5ad60738cdd927762f", + "Django-1.10.1-py2.py3-none-any.whl": "3d689905cd0635bbb33b87f9a5df7ca70a3db206faae4ec58cda5e7f5f47050d", + "celery-4.2.0-py2.py3-none-any.whl": "2082cbd82effa8ac8a8a58977d70bb203a9f362817e3b66f4578117b9f93d8a9", + "celery-4.1.1-py2.py3-none-any.whl": "6fc4678d1692af97e137b2a9f1c04efd8e7e2fb7134c5c5ad60738cdd927762f", } PYTHON_MD_PROJECT_SPECIFIER = [ @@ -159,9 +146,7 @@ # Intended to be used with the XS specifier PYTHON_WHEEL_FILENAME = "shelf_reader-0.1-py2-none-any.whl" -PYTHON_WHEEL_URL = urljoin( - urljoin(PYTHON_FIXTURES_URL, "packages/"), PYTHON_WHEEL_FILENAME -) +PYTHON_WHEEL_URL = urljoin(urljoin(PYTHON_FIXTURES_URL, "packages/"), PYTHON_WHEEL_FILENAME) PYTHON_WHEEL_SHA256 = "2eceb1643c10c5e4a65970baf63bde43b79cbdac7de81dae853ce47ab05197e9" PYTHON_XS_FIXTURE_CHECKSUMS = { @@ -269,7 +254,7 @@ "download_concurrency": 3, "policy": "on_demand", "prereleases": False, - "excludes": ["example1", "example2"] + "excludes": ["example1", "example2"], } BANDERSNATCH_CONF = b""" @@ -333,7 +318,7 @@ "requires_python": None, "size": 22455, "yanked": False, - "yanked_reason": None + "yanked_reason": None, } SHELF_SDIST_PYTHON_DOWNLOAD = { @@ -351,7 +336,7 @@ "requires_python": None, "size": 19097, "yanked": False, - "yanked_reason": None + "yanked_reason": None, } SHELF_0DOT1_RELEASE = [SHELF_BDIST_PYTHON_DOWNLOAD, SHELF_SDIST_PYTHON_DOWNLOAD] @@ -359,9 +344,6 @@ SHELF_PYTHON_JSON = { "info": PYTHON_INFO_DATA, "last_serial": 0, - "releases": { - "0.1": SHELF_0DOT1_RELEASE - }, - "urls": SHELF_0DOT1_RELEASE - + "releases": {"0.1": SHELF_0DOT1_RELEASE}, + "urls": SHELF_0DOT1_RELEASE, } diff --git a/pulp_python/tests/functional/utils.py b/pulp_python/tests/functional/utils.py index 619b9bbbe..e75fc4da6 100644 --- a/pulp_python/tests/functional/utils.py +++ b/pulp_python/tests/functional/utils.py @@ -1,6 +1,6 @@ -import requests - from urllib.parse import urljoin + +import requests from lxml import html @@ -16,6 +16,7 @@ def ensure_simple(simple_url, packages, sha_digests=None): in the simple index and thus be accessible from the distribution, but if one can't see it how would one know that it's there?* """ + def explore_links(page_url, page_name, links_found, msgs): legit_found_links = [] page = html.fromstring(requests.get(page_url).text) @@ -48,8 +49,16 @@ def explore_links(page_url, page_name, links_found, msgs): package = package_link.split("/")[-1] if sha_digests[package] != sha: msgs += f"\nRelease has bad sha256 attached to it {package}" - msgs += "".join(map(lambda x: f"\nSimple link not found for {x}", - [name for name, val in packages_found.items() if not val])) - msgs += "".join(map(lambda x: f"\nReleases link not found for {x}", - [name for name, val in releases_found.items() if not val])) + msgs += "".join( + map( + lambda x: f"\nSimple link not found for {x}", + [name for name, val in packages_found.items() if not val], + ) + ) + msgs += "".join( + map( + lambda x: f"\nReleases link not found for {x}", + [name for name, val in releases_found.items() if not val], + ) + ) return len(msgs) == 0, msgs diff --git a/pyproject.toml b/pyproject.toml index e4ccf4f09..b4e79c8a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,3 +124,30 @@ replace = "version = \"{new_version}\"" filename = "./pyproject.toml" search = "version = \"{current_version}\"" replace = "version = \"{new_version}\"" + + +[tool.ruff] +# This section is managed by the plugin template. Do not edit manually. +line-length = 100 +extend-exclude = [ + "docs/**", + "**/migrations/*.py", +] + +[tool.ruff.lint] +# This section is managed by the plugin template. Do not edit manually. +extend-select = [ + "I", + "INT", + "TID", + "T10", +] + +[tool.ruff.lint.flake8-tidy-imports.banned-api] +# This section is managed by the plugin template. Do not edit manually. +"pulpcore.app".msg = "The 'pulpcore' apis must only be consumed via 'pulpcore.plugin'." + +[tool.ruff.lint.isort] +# This section is managed by the plugin template. Do not edit manually. +sections = { second-party = ["pulpcore"] } +section-order = ["future", "standard-library", "third-party", "second-party", "first-party", "local-folder"] diff --git a/template_config.yml b/template_config.yml index 184774031..fad62f685 100644 --- a/template_config.yml +++ b/template_config.yml @@ -6,9 +6,7 @@ # After editing this file please always reapply the plugin template before committing any changes. --- -black: false check_commit_message: true -check_gettext: true check_manifest: true check_stray_pulpcore_imports: true ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" @@ -23,10 +21,9 @@ deploy_to_pypi: true disabled_redis_runners: [] docker_fixtures: false extra_files: [] -flake8: true -flake8_ignore: [] github_org: "pulp" latest_release_branch: "3.12" +lint_ignore: [] lint_requirements: true os_required_packages: [] parallel_test_workers: 8 @@ -82,7 +79,6 @@ pulp_settings_s3: BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" api_root: "/rerouted/djnd/" domain_enabled: true -pydocstyle: true release_email: "pulp-infra@redhat.com" release_user: "pulpbot" stalebot: true