diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index 4ef94c861cf..7826bc03420 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -76,6 +76,13 @@ retries: 2 delay: 5 + - name: "Wait for postgres-satellite" + ansible.builtin.wait_for: + host: "postgres-satellite" + port: 5432 + timeout: 30 + when: "multi_db_test | default(false)" + - name: "Wait for Pulp" ansible.builtin.uri: url: "http://pulp{{ pulp_scenario_settings.api_root | default(pulp_settings.api_root | default('\/pulp\/', True), True) }}api/v3/status/" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 612346b62ff..65abe1de11a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,7 +107,7 @@ jobs: uses: "./.github/workflows/test.yml" with: matrix_env: | - [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}, {"TEST": "multi_db"}] deprecations: runs-on: "ubuntu-latest" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index b759af34a72..9c9dc182869 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -123,6 +123,20 @@ pulp_scenario_env: {} VARSYAML fi +if [ "$TEST" = "multi_db" ]; then + cat >> .ci/ansible/vars/main.yaml << VARSYAML + - name: "postgres-satellite" + image: "docker.io/library/postgres:16" + env: + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "postgres" + POSTGRES_DB: "pulp" +multi_db_test: true +pulp_scenario_settings: null +pulp_scenario_env: {} +VARSYAML +fi + cat >> .ci/ansible/vars/main.yaml << VARSYAML ... VARSYAML diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 65ad9995e02..d0b548651ba 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -129,7 +129,11 @@ cmd_user_prefix bash -c "django-admin makemigrations file --check --dry-run" cmd_user_prefix bash -c "django-admin makemigrations certguard --check --dry-run" # Run unit tests. -cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --durations=20 --pyargs pulpcore.tests.unit" +MULTI_DB_ENV="" +if [[ "$TEST" == "multi_db" ]]; then + MULTI_DB_ENV="PULP_DATABASES__data_1__ENGINE=django.db.backends.postgresql PULP_DATABASES__data_1__NAME=pulp PULP_DATABASES__data_1__USER=postgres PULP_DATABASES__data_1__PASSWORD=postgres PULP_DATABASES__data_1__HOST=postgres-satellite PULP_DATABASES__data_1__PORT=5432 PULP_DATABASE_ROUTERS='[\"pulpcore.app.db_router.PulpDomainRouter\"]'" +fi +cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres $MULTI_DB_ENV pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --durations=20 --pyargs pulpcore.tests.unit" cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --durations=20 --pyargs pulp_file.tests.unit" cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --durations=20 --pyargs pulp_certguard.tests.unit" # Run functional tests diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index d90c373b68e..ffb512181f3 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -1,3 +1,4 @@ +import logging import random from collections import defaultdict from gettext import gettext as _ @@ -6,8 +7,8 @@ from django import apps from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.db import connection, transaction -from django.db.models.signals import post_migrate, pre_migrate +from django.db import connection, connections, transaction +from django.db.models.signals import post_delete, post_migrate, post_save, pre_migrate from django.utils.module_loading import module_has_submodule from pulpcore.exceptions.plugin import MissingPlugin @@ -255,14 +256,39 @@ def ready(self): post_migrate.connect( _populate_system_id, sender=self, dispatch_uid="populate_system_id_identifier" ) + post_migrate.connect( + _ensure_domains_replicated, + sender=self, + dispatch_uid="ensure_domains_replicated_identifier", + ) post_migrate.connect( _populate_artifact_serving_distribution, sender=self, dispatch_uid="populate_artifact_serving_distribution_identifier", ) + from pulpcore.app.domain_sync import on_domain_post_delete, on_domain_post_save + from pulpcore.app.models import Domain + + post_save.connect( + on_domain_post_save, sender=Domain, dispatch_uid="replicate_domain_post_save" + ) + post_delete.connect( + on_domain_post_delete, sender=Domain, dispatch_uid="replicate_domain_post_delete" + ) + + from pulpcore.app.db_router import is_multi_db_routing_active + + if is_multi_db_routing_active(): + from pulpcore.app.role_util import on_any_model_post_delete + + post_delete.connect( + on_any_model_post_delete, dispatch_uid="cleanup_cross_plane_roles_post_delete" + ) def _clean_app_status(sender, apps, verbosity, **kwargs): + if kwargs.get("using", "default") != "default": + return from django.contrib.postgres.functions import TransactionNow from django.db.models import F @@ -276,6 +302,9 @@ def _clean_app_status(sender, apps, verbosity, **kwargs): def _populate_access_policies(sender, apps, verbosity, **kwargs): + if kwargs.get("using", "default") != "default": + return + from pulpcore.app.util import get_view_urlpattern from pulpcore.app.viewsets import LoginViewSet @@ -320,12 +349,16 @@ def _populate_access_policies(sender, apps, verbosity, **kwargs): def _populate_system_id(sender, apps, verbosity, **kwargs): + if kwargs.get("using", "default") != "default": + return SystemID = apps.get_model("core", "SystemID") if not SystemID.objects.exists(): SystemID().save() def _ensure_default_domain(sender, **kwargs): + if kwargs.get("using", "default") != "default": + return table_names = connection.introspection.table_names() if "core_domain" in table_names: from pulpcore.app.util import get_default_domain @@ -343,7 +376,29 @@ def _ensure_default_domain(sender, **kwargs): default.save(skip_hooks=True) +def _ensure_domains_replicated(sender, **kwargs): + using = kwargs.get("using", "default") + if using == "default": + return + if "core_domain" not in connections[using].introspection.table_names(): + return + from pulpcore.app.domain_sync import reconcile_domains_to_alias + + try: + reconcile_domains_to_alias(using) + except Exception: + logging.getLogger(__name__).error( + "Reconciling Domain rows to alias '%s' failed during migration. Data-plane objects " + "created on this alias by later migrations/post_migrate hooks that FK to Domain may " + "fail until 'pulpcore-manager sync-domains' is run.", + using, + exc_info=True, + ) + + def _populate_roles(sender, apps, verbosity, **kwargs): + if kwargs.get("using", "default") != "default": + return role_prefix = f"{sender.label}." # collect all plugin defined roles desired_roles = {} @@ -403,6 +458,7 @@ def _get_permission(perm): def _populate_artifact_serving_distribution(sender, apps, verbosity, **kwargs): + alias = kwargs.get("using", "default") if ( settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem" or not settings.REDIRECT_TO_OBJECT_STORAGE @@ -415,15 +471,17 @@ def _populate_artifact_serving_distribution(sender, apps, verbosity, **kwargs): print(_("ArtifactDistribution model does not exist. Skipping initialization.")) return try: - ArtifactDistribution.objects.get() + ArtifactDistribution.objects.using(alias).get() except ArtifactDistribution.DoesNotExist: name = f"{random.getrandbits(256):x}" - with transaction.atomic(): - content_guard, _created = ContentRedirectContentGuard.objects.get_or_create( + with transaction.atomic(using=alias): + content_guard, _created = ContentRedirectContentGuard.objects.using( + alias + ).get_or_create( name=name, pulp_type="core.content_redirect", ) - _dist, _created = ArtifactDistribution.objects.get_or_create( + _dist, _created = ArtifactDistribution.objects.using(alias).get_or_create( name=name, pulp_type="core.artifact", defaults={"base_path": name, "content_guard": content_guard}, diff --git a/pulpcore/app/contexts.py b/pulpcore/app/contexts.py index d86b9ce6f11..a18c30191d1 100644 --- a/pulpcore/app/contexts.py +++ b/pulpcore/app/contexts.py @@ -12,6 +12,7 @@ current_pulp_api_version = ContextVar( "current_pulp_api_version", default=settings.REST_FRAMEWORK.get("DEFAULT_VERSION", "v3") ) +_current_migration_alias = ContextVar("current_migration_alias", default=None) @contextmanager @@ -45,6 +46,15 @@ def with_domain(domain): _current_domain.reset(token) +@contextmanager +def with_migration_alias(alias): + token = _current_migration_alias.set(alias) + try: + yield + finally: + _current_migration_alias.reset(token) + + @contextmanager def with_task_context(task): with with_domain(task.pulp_domain), with_guid(task.logging_cid), with_user(task.user): diff --git a/pulpcore/app/db_router.py b/pulpcore/app/db_router.py new file mode 100644 index 00000000000..509b31af099 --- /dev/null +++ b/pulpcore/app/db_router.py @@ -0,0 +1,86 @@ +import logging + +from django.apps import apps as django_apps +from django.db import router as django_router + +from pulpcore.app.contexts import _current_migration_alias +from pulpcore.app.util import get_domain + +logger = logging.getLogger(__name__) + +CONTROL_PLANE_LABELS = frozenset( + { + "core.domain", + "core.task", + "core.taskgroup", + "core.taskschedule", + "core.createdresource", + "core.appstatus", + "core.systemid", + "core.accesspolicy", + "core.role", + "core.userrole", + "core.grouprole", + "core.progressreport", + "core.groupprogressreport", + "core.migrationstatus", + "core.domainmove", + "core.profileartifact", + "core.signingservice", + "core.asciiarmoreddetachedsigningservice", + "container.manifestsigningservice", + "rpm.rpmpackagesigningservice", + } +) + +CONTROL_PLANE_APPS = frozenset({"auth", "contenttypes", "admin", "sessions"}) + + +def _database_alias(domain): + if "database_alias" in domain.__dict__: + return domain.__dict__["database_alias"] + return "default" + + +class PulpDomainRouter: + def _is_control_plane(self, model): + label = f"{model._meta.app_label}.{model._meta.model_name}" + return label in CONTROL_PLANE_LABELS or model._meta.app_label in CONTROL_PLANE_APPS + + def _resolve_db(self, model, **hints): + if model._meta.apps is not django_apps: + migration_alias = _current_migration_alias.get() + if migration_alias is not None: + return migration_alias + + if self._is_control_plane(model): + return "default" + + instance = hints.get("instance") + if instance is not None: + if "pulp_domain_id" in instance.__dict__: + domain = instance._state.fields_cache.get("pulp_domain") + if domain is not None: + return _database_alias(domain) + + domain = get_domain() + if domain is not None: + return _database_alias(domain) + + return "default" + + def db_for_read(self, model, **hints): + return self._resolve_db(model, **hints) + + def db_for_write(self, model, **hints): + return self._resolve_db(model, **hints) + + def allow_relation(self, obj1, obj2, **hints): + return True + + def allow_migrate(self, db, app_label, model_name=None, **hints): + return True + + +def is_multi_db_routing_active(): + return any(isinstance(r, PulpDomainRouter) for r in django_router.routers) diff --git a/pulpcore/app/domain_move.py b/pulpcore/app/domain_move.py new file mode 100644 index 00000000000..f04e38ecf51 --- /dev/null +++ b/pulpcore/app/domain_move.py @@ -0,0 +1,203 @@ +import logging +from contextlib import contextmanager + +from django.apps import apps as django_apps +from django.db import IntegrityError, connections +from django.db.models import ProtectedError, RestrictedError +from django.db.models.fields.files import FileField +from django_lifecycle.mixins import LifecycleModelMixin + +from pulpcore.app.contexts import with_domain +from pulpcore.app.db_router import PulpDomainRouter + +logger = logging.getLogger(__name__) + +_router = PulpDomainRouter() + +THROUGH_MODEL_DOMAIN_FILTERS = { + "core.contentartifact": "content__pulp_domain_id", + "core.repositorycontent": "repository__pulp_domain_id", + "core.repositoryversion": "repository__pulp_domain_id", + "core.repositoryversioncontentdetails": "repository_version__repository__pulp_domain_id", + "core.publishedartifact": "publication__pulp_domain_id", + "core.distributedpublication": "distribution__pulp_domain_id", + "core.alternatecontentsourcepath": "alternate_content_source__pulp_domain_id", + "core.pulpimporterrepository": "repository__pulp_domain_id", + "core.uploadchunk": "upload__pulp_domain_id", + "core.exportedresource": "export__pulp_domain_id", + "container.blobmanifest": "manifest__pulp_domain_id", + "container.manifestlistmanifest": "manifest_list__pulp_domain_id", + "rpm.addon": "distribution_tree__pulp_domain_id", + "rpm.checksum": "distribution_tree__pulp_domain_id", + "rpm.image": "distribution_tree__pulp_domain_id", + "rpm.variant": "distribution_tree__pulp_domain_id", + "rpm.rpmpackagesigningresult": "result_package__pulp_domain_id", + "rpm.updatecollection": "update_record__pulp_domain_id", + "rpm.updatereference": "update_record__pulp_domain_id", + "rpm.updatecollectionpackage": "update_collection__update_record__pulp_domain_id", + "python.pythonblocklistentry": "repository__pulp_domain_id", +} + + +class DomainMoveError(Exception): + pass + + +def data_plane_models(): + models = [] + for model in django_apps.get_models(): + if model._meta.proxy or model._meta.auto_created: + continue + if _router._is_control_plane(model): + continue + if hasattr(model, "pulp_domain_id"): + models.append((model, "pulp_domain_id")) + for label, lookup in THROUGH_MODEL_DOMAIN_FILTERS.items(): + try: + model = django_apps.get_model(label) + except LookupError: + continue + models.append((model, lookup)) + return models + + +def _domain_queryset(model, lookup, alias, domain): + return model.objects.using(alias).filter(**{lookup: domain.pk}) + + +def estimate_domain_size(domain, alias): + rows = [] + for model, lookup in data_plane_models(): + count = _domain_queryset(model, lookup, alias, domain).count() + table = model._meta.db_table + with connections[alias].cursor() as cursor: + cursor.execute("SELECT pg_total_relation_size(%s)", [table]) + (table_size,) = cursor.fetchone() + rows.append( + { + "model": model._meta.label, + "table": table, + "row_count": count, + "table_total_size_bytes": table_size or 0, + } + ) + return rows + + +def _run_passes(models, action, action_description): + remaining = list(models) + results = {} + while remaining: + blocked = [] + progressed = False + for model, lookup in remaining: + try: + results[model._meta.label] = action(model, lookup) + except (IntegrityError, ProtectedError, RestrictedError): + blocked.append((model, lookup)) + else: + progressed = True + if not progressed: + raise DomainMoveError( + f"Could not {action_description} for: " + f"{', '.join(model._meta.label for model, _ in blocked)} " + f"(unresolved FK dependency -- see data_plane_models()/" + f"THROUGH_MODEL_DOMAIN_FILTERS if this is a plugin model this module doesn't " + f"know about)." + ) + remaining = blocked + return results + + +def _copy_model(model, lookup, domain, source_alias, target_alias): + fields = model._meta.concrete_fields + copied = 0 + for row in _domain_queryset(model, lookup, source_alias, domain).iterator(): + values = {} + for field in fields: + value = getattr(row, field.attname) + if isinstance(field, FileField) and value: + value = value.name + values[field.attname] = value + instance = model(**values) + instance._state.adding = False + if isinstance(instance, LifecycleModelMixin): + instance.save(using=target_alias, skip_hooks=True) + else: + instance.save(using=target_alias) + copied += 1 + return copied + + +def copy_domain_data(domain, source_alias, target_alias): + with with_domain(domain): + return _run_passes( + data_plane_models(), + lambda model, lookup: _copy_model(model, lookup, domain, source_alias, target_alias), + "copy data", + ) + + +def _row_checksum(pks): + import hashlib + + return hashlib.sha256(",".join(sorted(str(pk) for pk in pks)).encode()).hexdigest() + + +def verify_domain_data(domain, source_alias, target_alias): + mismatches = [] + for model, lookup in data_plane_models(): + source_pks = list( + _domain_queryset(model, lookup, source_alias, domain).values_list("pk", flat=True) + ) + target_pks = list( + _domain_queryset(model, lookup, target_alias, domain).values_list("pk", flat=True) + ) + source_checksum = _row_checksum(source_pks) + target_checksum = _row_checksum(target_pks) + if len(source_pks) != len(target_pks) or source_checksum != target_checksum: + mismatches.append( + { + "model": model._meta.label, + "source_count": len(source_pks), + "target_count": len(target_pks), + "source_checksum": source_checksum, + "target_checksum": target_checksum, + } + ) + return mismatches + + +def _delete_model(model, lookup, domain, alias): + return _domain_queryset(model, lookup, alias, domain).delete()[0] + + +def delete_domain_data(domain, alias): + return _run_passes( + data_plane_models(), + lambda model, lookup: _delete_model(model, lookup, domain, alias), + "delete data", + ) + + +@contextmanager +def _advisory_lock(lock_id, error_message): + with connections["default"].cursor() as cursor: + cursor.execute("SELECT pg_try_advisory_lock(%s)", [lock_id]) + (acquired,) = cursor.fetchone() + if not acquired: + raise DomainMoveError(error_message) + try: + yield + finally: + cursor.execute("SELECT pg_advisory_unlock(%s)", [lock_id]) + + +def domain_move_lock(): + from pulpcore.constants import DOMAIN_MOVE_LOCK + + return _advisory_lock( + DOMAIN_MOVE_LOCK, + "Could not acquire the domain-move advisory lock. Another 'move-domain' run is " + "already in progress.", + ) diff --git a/pulpcore/app/domain_sync.py b/pulpcore/app/domain_sync.py new file mode 100644 index 00000000000..36dd7a18a69 --- /dev/null +++ b/pulpcore/app/domain_sync.py @@ -0,0 +1,171 @@ +import logging +import time + +from django.conf import settings + +logger = logging.getLogger(__name__) + +REPLICATION_RETRY_ATTEMPTS = 3 +REPLICATION_RETRY_BACKOFF = 1 + + +def satellite_aliases(): + return [alias for alias in settings.DATABASES if alias != "default"] + + +def _target_aliases(domain): + if domain.name == "default": + return satellite_aliases() + if domain.database_alias in satellite_aliases(): + return [domain.database_alias] + return [] + + +def domain_field_values(domain): + return {field.attname: getattr(domain, field.attname) for field in domain._meta.concrete_fields} + + +def _comparable_domain_field_values(domain): + values = domain_field_values(domain) + values.pop("pulp_last_updated", None) + return values + + +def replicate_domain_save(domain, using=None, attempts=REPLICATION_RETRY_ATTEMPTS): + values = domain_field_values(domain) + pulp_id = values.pop("pulp_id") + for alias in _target_aliases(domain): + if alias == using: + continue + _replicate_one_save(alias, pulp_id, values, attempts) + + +def replicate_domain_delete(domain, using=None, attempts=REPLICATION_RETRY_ATTEMPTS): + for alias in _target_aliases(domain): + if alias == using: + continue + _replicate_one_delete(alias, domain.pulp_id, attempts) + + +def ensure_domain_on_alias(domain, alias, attempts=REPLICATION_RETRY_ATTEMPTS): + values = domain_field_values(domain) + pulp_id = values.pop("pulp_id") + _replicate_one_save(alias, pulp_id, values, attempts) + + +def reconcile_domains_to_alias(alias, dry_run=False): + from pulpcore.app.models import Domain + + desired_domains = { + domain.pulp_id: domain + for domain in Domain.objects.using("default") + if domain.name == "default" or domain.database_alias == alias + } + desired_ids = set(desired_domains) + + satellite_ids = set(Domain.objects.using(alias).values_list("pulp_id", flat=True)) + + missing = desired_ids - satellite_ids + extra = satellite_ids - desired_ids + stale = set() + for pulp_id in desired_ids & satellite_ids: + satellite_domain = Domain.objects.using(alias).get(pulp_id=pulp_id) + if _comparable_domain_field_values( + desired_domains[pulp_id] + ) != _comparable_domain_field_values(satellite_domain): + stale.add(pulp_id) + + if dry_run: + return {"missing": missing, "extra": extra, "stale": stale} + + for pulp_id in missing | stale: + values = domain_field_values(desired_domains[pulp_id]) + values.pop("pulp_id") + try: + instance = Domain.objects.using(alias).get(pulp_id=pulp_id) + for key, value in values.items(): + setattr(instance, key, value) + except Domain.DoesNotExist: + instance = Domain(pulp_id=pulp_id, **values) + instance.save(using=alias, skip_hooks=True) + for pulp_id in extra: + Domain.objects.using(alias).filter(pulp_id=pulp_id).delete() + + return {"missing": missing, "extra": extra, "stale": stale} + + +def _replicate_one_save(alias, pulp_id, defaults, attempts): + from pulpcore.app.models import Domain + + delay = REPLICATION_RETRY_BACKOFF + for attempt in range(1, attempts + 1): + try: + manager = Domain.objects.using(alias) + try: + instance = manager.get(pulp_id=pulp_id) + for key, value in defaults.items(): + setattr(instance, key, value) + except Domain.DoesNotExist: + instance = Domain(pulp_id=pulp_id, **defaults) + instance.save(using=alias, skip_hooks=True) + return + except Exception: + logger.warning( + "Domain replication to alias '%s' failed (attempt %d/%d) for domain %s.", + alias, + attempt, + attempts, + pulp_id, + exc_info=True, + ) + if attempt < attempts: + time.sleep(delay) + delay *= 2 + logger.error( + "Domain replication to alias '%s' failed after %d attempts for domain %s. " + "Run 'pulpcore-manager sync-domains' to reconcile.", + alias, + attempts, + pulp_id, + ) + + +def _replicate_one_delete(alias, pulp_id, attempts): + from pulpcore.app.models import Domain + + delay = REPLICATION_RETRY_BACKOFF + for attempt in range(1, attempts + 1): + try: + Domain.objects.using(alias).filter(pulp_id=pulp_id).delete() + return + except Exception: + logger.warning( + "Domain delete-replication to alias '%s' failed (attempt %d/%d) for domain %s.", + alias, + attempt, + attempts, + pulp_id, + exc_info=True, + ) + if attempt < attempts: + time.sleep(delay) + delay *= 2 + logger.error( + "Domain delete-replication to alias '%s' failed after %d attempts for domain %s. " + "Run 'pulpcore-manager sync-domains' to reconcile.", + alias, + attempts, + pulp_id, + ) + + +def on_domain_post_save(sender, instance, created, using, **kwargs): + if using != "default": + return + replicate_domain_save(instance, using=using) + + +def on_domain_post_delete(sender, instance, using, **kwargs): + if using != "default": + return + replicate_domain_delete(instance, using=using) diff --git a/pulpcore/app/management/commands/analyze-publication.py b/pulpcore/app/management/commands/analyze-publication.py index 07252c96e2d..f796aae91db 100644 --- a/pulpcore/app/management/commands/analyze-publication.py +++ b/pulpcore/app/management/commands/analyze-publication.py @@ -3,7 +3,7 @@ from django.core.management import BaseCommand, CommandError from django.urls import reverse -from pulpcore.app.models import Artifact, Distribution, Publication +from pulpcore.app.models import Artifact, Distribution, Domain, Publication from pulpcore.app.util import get_view_name_for_model @@ -19,6 +19,12 @@ def add_arguments(self, parser): "--distribution-base-path", required=False, help=_("A base_path of a distribution.") ) parser.add_argument("--tabular", action="store_true", help=_("Display as a table")) + parser.add_argument( + "--domain", + default="default", + required=False, + help=_("The pulp domain the publication/distribution belongs to."), + ) def handle(self, *args, **options): """Implement the command.""" @@ -33,17 +39,28 @@ def handle(self, *args, **options): raise CommandError("Must provide either --publication or --distribution-base-path") elif options["publication"] and options["distribution_base_path"]: raise CommandError("Cannot provide both --publication and --distribution-base-path") - elif options["publication"]: - publication = Publication.objects.get(pk=options["publication"]) + + try: + domain = Domain.objects.get(name=options["domain"]) + except Domain.DoesNotExist: + raise CommandError(_("Domain '{name}' does not exist.").format(name=options["domain"])) + alias = domain.database_alias + + if options["publication"]: + publication = Publication.objects.using(alias).get(pk=options["publication"]) else: - distribution = Distribution.objects.get(base_path=options["distribution_base_path"]) + distribution = Distribution.objects.using(alias).get( + base_path=options["distribution_base_path"] + ) if distribution.publication: publication = distribution.publication elif distribution.repository: repository = distribution.repository - publication = Publication.objects.filter( - repository_version__in=repository.versions.all(), complete=True - ).latest("repository_version", "pulp_created") + publication = ( + Publication.objects.using(alias) + .filter(repository_version__in=repository.versions.all(), complete=True) + .latest("repository_version", "pulp_created") + ) published_artifacts = publication.published_artifact.select_related( "content_artifact__artifact" diff --git a/pulpcore/app/management/commands/cleanup-moved-domain.py b/pulpcore/app/management/commands/cleanup-moved-domain.py new file mode 100644 index 00000000000..c14803442fd --- /dev/null +++ b/pulpcore/app/management/commands/cleanup-moved-domain.py @@ -0,0 +1,134 @@ +from gettext import gettext as _ + +from django.core.management import BaseCommand, CommandError +from django.utils.timezone import now + +from pulpcore.app.domain_move import DomainMoveError, delete_domain_data +from pulpcore.app.models import Domain, DomainMove + + +class Command(BaseCommand): + """ + Delete a moved domain's stale data left behind on its previous database alias. + + Run this after 'move-domain' has completed and the monitoring window has passed. + This permanently deletes data and cannot be undone, so it requires --force. + """ + + help = __doc__ + + def add_arguments(self, parser): + parser.add_argument("domain", help=_("Name of the previously-moved domain to clean up.")) + parser.add_argument( + "--from", + dest="from_alias", + help=_( + "The alias to delete the domain's stale rows from. Defaults to the " + "`from_alias` of the domain's most recent completed DomainMove record onto its " + "current alias. Required if no such record exists (e.g. the domain was moved " + "by means other than 'move-domain')." + ), + ) + parser.add_argument( + "--force", + action="store_true", + help=_( + "Required. Explicit acknowledgement that this permanently deletes data from " + "'--from' with no way to roll back afterwards." + ), + ) + + def handle(self, *args, **options): + try: + domain = Domain.objects.using("default").get(name=options["domain"]) + except Domain.DoesNotExist: + raise CommandError(_("No domain named '{name}' exists.").format(name=options["domain"])) + + if domain.moving: + raise CommandError( + _( + "Domain '{name}' has moving=True -- a move is in progress. Wait for it to " + "finish (or fail cleanly) before cleaning up." + ).format(name=domain.name) + ) + if domain.database_alias == "default": + raise CommandError( + _( + "Domain '{name}' is currently on 'default' -- nothing to clean up (either " + "it was never moved, or it was already moved back)." + ).format(name=domain.name) + ) + + move = ( + DomainMove.objects.using("default") + .filter(domain=domain, status="completed", to_alias=domain.database_alias) + .order_by("-cutover_at") + .first() + ) + + from_alias = options["from_alias"] or (move and move.from_alias) + if not from_alias: + raise CommandError( + _( + "No completed DomainMove record found for domain '{name}' onto its current " + "alias '{alias}'. Pass --from explicitly (the alias to delete the domain's " + "stale data from) if this domain was moved by means other than " + "'move-domain'." + ).format(name=domain.name, alias=domain.database_alias) + ) + if from_alias == domain.database_alias: + raise CommandError( + _( + "--from ('{alias}') is the domain's current alias; refusing to clean that up." + ).format(alias=from_alias) + ) + + if move and move.monitoring_until and now() < move.monitoring_until: + self.stdout.write( + self.style.WARNING( + _( + "The recommended monitoring window for this move does not end until " + "{until}. Proceeding anyway since you're running this command, but " + "consider waiting." + ).format(until=move.monitoring_until) + ) + ) + + if not options["force"]: + raise CommandError( + _( + "Refusing to delete domain '{name}''s data from '{alias}' without --force. " + "This is permanent and cannot be rolled back afterwards -- re-run with " + "--force once you are certain." + ).format(name=domain.name, alias=from_alias) + ) + + self.stdout.write( + _("Deleting domain '{name}''s data from '{alias}'...").format( + name=domain.name, alias=from_alias + ) + ) + try: + deleted = delete_domain_data(domain, from_alias) + except DomainMoveError as e: + raise CommandError(str(e)) from e + + for label, count in deleted.items(): + if count: + self.stdout.write(f" {label}: {count} row(s) deleted") + + if from_alias != "default": + Domain.objects.using(from_alias).filter(pulp_id=domain.pulp_id).delete() + + if move: + move.cleaned_up_at = now() + move.status = "cleaned_up" + move.save(update_fields=["cleaned_up_at", "status"]) + + self.stdout.write( + self.style.SUCCESS( + _("Domain '{name}''s data removed from '{alias}'.").format( + name=domain.name, alias=from_alias + ) + ) + ) diff --git a/pulpcore/app/management/commands/datarepair-2327.py b/pulpcore/app/management/commands/datarepair-2327.py index 56a59bab969..46af36b57cf 100644 --- a/pulpcore/app/management/commands/datarepair-2327.py +++ b/pulpcore/app/management/commands/datarepair-2327.py @@ -3,11 +3,12 @@ import cryptography from django.conf import settings from django.core.management import BaseCommand -from django.db import connection +from django.db import connections from django.db.models import Q from django.utils.encoding import force_bytes, force_str from pulpcore.app.models import Remote +from pulpcore.app.util import for_each_domain class Command(BaseCommand): @@ -45,64 +46,71 @@ def handle(self, *args, **options): | Q(client_key__isnull=False) ) - number_unencrypted = 0 - number_multi_encrypted = 0 - - for remote_pk in Remote.objects.filter(possibly_affected_remotes).values_list( - "pk", flat=True - ): - try: - remote = Remote.objects.get(pk=remote_pk) - # if we can get the remote successfully, it is either OK or the fields are - # encrypted more than once - except cryptography.fernet.InvalidToken: - # If decryption fails then it probably hasn't been encrypted yet - # get the raw column value, avoiding any Django field handling - with connection.cursor() as cursor: - cursor.execute( - "SELECT username, password, proxy_username, proxy_password, client_key " - "FROM core_remote WHERE pulp_id = %s", - [str(remote_pk)], - ) - row = cursor.fetchone() - - field_values = {} + counts = {"number_unencrypted": 0, "number_multi_encrypted": 0} + + def _repair_for_domain(domain, alias): + for remote_pk in ( + Remote.objects.using(alias) + .filter(possibly_affected_remotes) + .values_list("pk", flat=True) + ): + try: + remote = Remote.objects.using(alias).get(pk=remote_pk) + # if we can get the remote successfully, it is either OK or the fields are + # encrypted more than once + except cryptography.fernet.InvalidToken: + # If decryption fails then it probably hasn't been encrypted yet + # get the raw column value, avoiding any Django field handling + with connections[alias].cursor() as cursor: + cursor.execute( + "SELECT username, password, proxy_username, proxy_password, " + "client_key FROM core_remote WHERE pulp_id = %s", + [str(remote_pk)], + ) + row = cursor.fetchone() + + field_values = {} + + for field, value in zip(fields, row): + field_values[field] = value - for field, value in zip(fields, row): - field_values[field] = value - - if not dry_run: - Remote.objects.filter(pk=remote_pk).update(**field_values) - number_unencrypted += 1 - else: - times_decrypted = 0 - keep_trying = True - needs_update = False - - while keep_trying: - for field in fields: - field_value = getattr(remote, field) # value gets decrypted once on access - if not field_value: - continue - - try: - # try to decrypt it again - field_value = force_str(fernet.decrypt(force_bytes(field_value))) - # it was decrypted successfully again time, so it was probably - # encrypted multiple times over. lets re-set the value with the - # newly decrypted value - setattr(remote, field, field_value) - needs_update = True - except cryptography.fernet.InvalidToken: - # couldn't be decrypted again, stop here - keep_trying = False - - times_decrypted += 1 - - if needs_update: if not dry_run: - remote.save() - number_multi_encrypted += 1 + Remote.objects.using(alias).filter(pk=remote_pk).update(**field_values) + counts["number_unencrypted"] += 1 + else: + times_decrypted = 0 + keep_trying = True + needs_update = False + + while keep_trying: + for field in fields: + # value gets decrypted once on access + field_value = getattr(remote, field) + if not field_value: + continue + + try: + # try to decrypt it again + field_value = force_str(fernet.decrypt(force_bytes(field_value))) + # it was decrypted successfully again time, so it was probably + # encrypted multiple times over. lets re-set the value with the + # newly decrypted value + setattr(remote, field, field_value) + needs_update = True + except cryptography.fernet.InvalidToken: + # couldn't be decrypted again, stop here + keep_trying = False + + times_decrypted += 1 + + if needs_update: + if not dry_run: + remote.save() + counts["number_multi_encrypted"] += 1 + + for_each_domain(_repair_for_domain) + number_unencrypted = counts["number_unencrypted"] + number_multi_encrypted = counts["number_multi_encrypted"] if dry_run: print("Remotes with un-encrypted fields: {}".format(number_unencrypted)) diff --git a/pulpcore/app/management/commands/datarepair.py b/pulpcore/app/management/commands/datarepair.py index ab998f45752..d561723da85 100644 --- a/pulpcore/app/management/commands/datarepair.py +++ b/pulpcore/app/management/commands/datarepair.py @@ -3,11 +3,12 @@ import cryptography from django.conf import settings from django.core.management import BaseCommand, CommandError -from django.db import connection +from django.db import connections from django.db.models import Q from django.utils.encoding import force_bytes, force_str from pulpcore.app import models +from pulpcore.app.util import domain_db, for_each_domain class Command(BaseCommand): @@ -50,57 +51,62 @@ def repair_7272(self, options): for domain in models.Domain.objects.all(): has_printed_domain = False - for repo in models.Repository.objects.filter(pulp_domain=domain): - for rv in models.RepositoryVersion.objects.filter(repository=repo): - needs_fix = False - if rv.content_ids is not None: - cached_id_set = set(rv.content_ids) - repositorycontent_id_set = set( - rv._content_relationships().values_list("content__pk", flat=True) + with domain_db(domain) as alias: + for repo in models.Repository.objects.using(alias).filter(pulp_domain=domain): + for rv in models.RepositoryVersion.objects.using(alias).filter(repository=repo): + needs_fix = False + if rv.content_ids is not None: + cached_id_set = set(rv.content_ids) + repositorycontent_id_set = set( + rv._content_relationships().values_list("content__pk", flat=True) + ) + if cached_id_set != repositorycontent_id_set: + if not has_printed_domain: + self.stdout.write(f'In domain "{domain.name}"') + has_printed_domain = True + + self.stdout.write( + f'\tRepository "{repo.name}" (type "{repo.pulp_type}") ' + f"version {rv.number} has a mismatch between the " + "RepositoryContent and the cached ID set" + ) + needs_fix = True + + repositorycontent_id_count = rv._content_relationships().count() + if repositorycontent_id_count == 0: + continue + rv_count_details = models.RepositoryVersionContentDetails.objects.using( + alias + ).filter( + repository_version=rv, + count_type=models.RepositoryVersionContentDetails.PRESENT, ) - if cached_id_set != repositorycontent_id_set: + + # need to sum across all content types + total_count = sum(rvcd.count for rvcd in rv_count_details) + + if total_count != repositorycontent_id_count: + needs_fix = True if not has_printed_domain: self.stdout.write(f'In domain "{domain.name}"') has_printed_domain = True - self.stdout.write( f'\tRepository "{repo.name}" (type "{repo.pulp_type}") ' f"version {rv.number} has a mismatch between the " - "RepositoryContent and the cached ID set" + "RepositoryContent and RepositoryVersionContentDetails" ) - needs_fix = True - repositorycontent_id_count = rv._content_relationships().count() - if repositorycontent_id_count == 0: - continue - rv_count_details = models.RepositoryVersionContentDetails.objects.filter( - repository_version=rv, - count_type=models.RepositoryVersionContentDetails.PRESENT, - ) - - # need to sum across all content types - total_count = sum(rvcd.count for rvcd in rv_count_details) - - if total_count != repositorycontent_id_count: - needs_fix = True - if not has_printed_domain: - self.stdout.write(f'In domain "{domain.name}"') - has_printed_domain = True - self.stdout.write( - f'\tRepository "{repo.name}" (type "{repo.pulp_type}") ' - f"version {rv.number} has a mismatch between the " - "RepositoryContent and RepositoryVersionContentDetails" - ) - - if needs_fix: - number_broken += 1 + if needs_fix: + number_broken += 1 - if not dry_run: - rv.content_ids = list( - rv._content_relationships().values_list("content__pk", flat=True) - ) - rv.save() - rv._compute_counts() + if not dry_run: + rv.content_ids = list( + rv._content_relationships().values_list( + "content__pk", flat=True + ) + ) + rv.save() + rv._compute_counts() self.stdout.write() @@ -130,64 +136,73 @@ def repair_2327(self, options): | Q(client_key__isnull=False) ) - number_unencrypted = 0 - number_multi_encrypted = 0 - - for remote_pk in models.Remote.objects.filter(possibly_affected_remotes).values_list( - "pk", flat=True - ): - try: - remote = models.Remote.objects.get(pk=remote_pk) - # if we can get the remote successfully, it is either OK or the fields are - # encrypted more than once - except cryptography.fernet.InvalidToken: - # If decryption fails then it probably hasn't been encrypted yet - # get the raw column value, avoiding any Django field handling - with connection.cursor() as cursor: - cursor.execute( - "SELECT username, password, proxy_username, proxy_password, client_key " - "FROM core_remote WHERE pulp_id = %s", - [str(remote_pk)], - ) - row = cursor.fetchone() - - field_values = {} + counts = {"number_unencrypted": 0, "number_multi_encrypted": 0} + + def _repair_2327_for_domain(domain, alias): + for remote_pk in ( + models.Remote.objects.using(alias) + .filter(possibly_affected_remotes) + .values_list("pk", flat=True) + ): + try: + remote = models.Remote.objects.using(alias).get(pk=remote_pk) + # if we can get the remote successfully, it is either OK or the fields are + # encrypted more than once + except cryptography.fernet.InvalidToken: + # If decryption fails then it probably hasn't been encrypted yet + # get the raw column value, avoiding any Django field handling + with connections[alias].cursor() as cursor: + cursor.execute( + "SELECT username, password, proxy_username, proxy_password, " + "client_key FROM core_remote WHERE pulp_id = %s", + [str(remote_pk)], + ) + row = cursor.fetchone() - for field, value in zip(fields, row): - field_values[field] = value + field_values = {} - if not dry_run: - models.Remote.objects.filter(pk=remote_pk).update(**field_values) - number_unencrypted += 1 - else: - times_decrypted = 0 - keep_trying = True - needs_update = False - - while keep_trying: - for field in fields: - field_value = getattr(remote, field) # value gets decrypted once on access - if not field_value: - continue + for field, value in zip(fields, row): + field_values[field] = value - try: - # try to decrypt it again - field_value = force_str(fernet.decrypt(force_bytes(field_value))) - # it was decrypted successfully again time, so it was probably - # encrypted multiple times over. lets re-set the value with the - # newly decrypted value - setattr(remote, field, field_value) - needs_update = True - except cryptography.fernet.InvalidToken: - # couldn't be decrypted again, stop here - keep_trying = False - - times_decrypted += 1 - - if needs_update: if not dry_run: - remote.save() - number_multi_encrypted += 1 + models.Remote.objects.using(alias).filter(pk=remote_pk).update( + **field_values + ) + counts["number_unencrypted"] += 1 + else: + times_decrypted = 0 + keep_trying = True + needs_update = False + + while keep_trying: + for field in fields: + # value gets decrypted once on access + field_value = getattr(remote, field) + if not field_value: + continue + + try: + # try to decrypt it again + field_value = force_str(fernet.decrypt(force_bytes(field_value))) + # it was decrypted successfully again time, so it was probably + # encrypted multiple times over. lets re-set the value with the + # newly decrypted value + setattr(remote, field, field_value) + needs_update = True + except cryptography.fernet.InvalidToken: + # couldn't be decrypted again, stop here + keep_trying = False + + times_decrypted += 1 + + if needs_update: + if not dry_run: + remote.save() + counts["number_multi_encrypted"] += 1 + + for_each_domain(_repair_2327_for_domain) + number_unencrypted = counts["number_unencrypted"] + number_multi_encrypted = counts["number_multi_encrypted"] if dry_run: print("Remotes with un-encrypted fields: {}".format(number_unencrypted)) @@ -210,22 +225,25 @@ def repair_7465(self, options): for domain in models.Domain.objects.all(): has_printed_domain = False - for repo in models.Repository.objects.filter(pulp_domain=domain): - for rv in models.RepositoryVersion.objects.filter(repository=repo): - if rv.content_ids is None: - if not has_printed_domain: - self.stdout.write(f'In domain "{domain.name}"') - has_printed_domain = True - number_missing += 1 - self.stdout.write( - f'\tRepository "{repo.name}" (type "{repo.pulp_type}") ' - f"version {rv.number} has a missing content_ids cache" - ) - if not dry_run: - rv.content_ids = list( - rv._content_relationships().values_list("content__pk", flat=True) + with domain_db(domain) as alias: + for repo in models.Repository.objects.using(alias).filter(pulp_domain=domain): + for rv in models.RepositoryVersion.objects.using(alias).filter(repository=repo): + if rv.content_ids is None: + if not has_printed_domain: + self.stdout.write(f'In domain "{domain.name}"') + has_printed_domain = True + number_missing += 1 + self.stdout.write( + f'\tRepository "{repo.name}" (type "{repo.pulp_type}") ' + f"version {rv.number} has a missing content_ids cache" ) - rv.save() + if not dry_run: + rv.content_ids = list( + rv._content_relationships().values_list( + "content__pk", flat=True + ) + ) + rv.save() if not number_missing: self.stdout.write("Finished. (OK)") diff --git a/pulpcore/app/management/commands/domain-size.py b/pulpcore/app/management/commands/domain-size.py new file mode 100644 index 00000000000..c57f21395d9 --- /dev/null +++ b/pulpcore/app/management/commands/domain-size.py @@ -0,0 +1,51 @@ +from gettext import gettext as _ + +from django.core.management import BaseCommand, CommandError + +from pulpcore.app.domain_move import estimate_domain_size +from pulpcore.app.models import Domain + + +class Command(BaseCommand): + """ + Report per-model row counts and on-disk table sizes for a domain's data. + + Reports on the domain's data-plane objects on its current database alias. + """ + + help = __doc__ + + def add_arguments(self, parser): + parser.add_argument("domain", help=_("Name of the domain to report on.")) + + def handle(self, *args, **options): + try: + domain = Domain.objects.using("default").get(name=options["domain"]) + except Domain.DoesNotExist: + raise CommandError(_("No domain named '{name}' exists.").format(name=options["domain"])) + + self.stdout.write( + _("Domain '{name}' (currently on alias '{alias}'):").format( + name=domain.name, alias=domain.database_alias + ) + ) + report = estimate_domain_size(domain, domain.database_alias) + total_rows = 0 + any_rows = False + for row in report: + if row["row_count"] == 0: + continue + any_rows = True + total_rows += row["row_count"] + self.stdout.write( + " {model}: {rows} row(s) (table '{table}' total size: {size} bytes)".format( + model=row["model"], + rows=row["row_count"], + table=row["table"], + size=row["table_total_size_bytes"], + ) + ) + if not any_rows: + self.stdout.write(_(" No data-plane rows found for this domain.")) + else: + self.stdout.write(_("Total rows across all models: {n}").format(n=total_rows)) diff --git a/pulpcore/app/management/commands/dump-publications-to-fs.py b/pulpcore/app/management/commands/dump-publications-to-fs.py index 299b0c085b3..5146947e839 100644 --- a/pulpcore/app/management/commands/dump-publications-to-fs.py +++ b/pulpcore/app/management/commands/dump-publications-to-fs.py @@ -5,12 +5,13 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.management import BaseCommand, CommandError -from pulpcore.app.models import Distribution, Publication +from pulpcore.app.models import Distribution, Domain, Publication from pulpcore.app.tasks.export import ( UnexportableArtifactException, _export_location_is_clean, _export_publication_to_file_system, ) +from pulpcore.app.util import for_each_domain from pulpcore.app.viewsets.base import NamedModelViewSet from pulpcore.constants import FS_EXPORT_METHODS @@ -23,6 +24,12 @@ class Command(BaseCommand): def add_arguments(self, parser): """Set up arguments.""" parser.add_argument("--publication", required=False, help=_("A publication ID.")) + parser.add_argument( + "--domain", + default="default", + required=False, + help=_("The pulp domain --publication belongs to (ignored otherwise)."), + ) parser.add_argument( "--distribution-path-prefix", required=False, @@ -75,45 +82,53 @@ def handle(self, *args, **options): publication_pk = NamedModelViewSet.extract_pk(options["publication"]) except Exception: publication_pk = options["publication"] - publication = Publication.objects.get(pk=publication_pk) + try: + domain = Domain.objects.get(name=options["domain"]) + except Domain.DoesNotExist: + raise CommandError( + _("Domain '{name}' does not exist.").format(name=options["domain"]) + ) + publication = Publication.objects.using(domain.database_alias).get(pk=publication_pk) to_export.append((options["dest"], publication)) # If no publication was specified go through the distributions and dump them if they # meet the criteria else: - # If a base_path prefix was provided, filter out distributions with a base path - # that doesn't start with the prefix - if options.get("distribution_path_prefix"): - distributions = Distribution.objects.filter( - base_path__startswith=options["distribution_path_prefix"] - ) - else: - distributions = Distribution.objects.all() - - # Filter out distributions that don't match the type specified (if any) - if options["type"]: - distributions = distributions.filter(pulp_type__startswith=options["type"]) - - # For all matching distributions, if they have a publication, dump it in a directory - # matching the original distribution structure - for distribution in distributions: - if distribution.publication: - publication = distribution.publication - elif distribution.repository: - repository = distribution.repository - # Account for distributions serving the latest publication of a given repository - try: - publication = Publication.objects.filter( - repository_version__in=repository.versions.all(), complete=True - ).latest("repository_version", "pulp_created") - repo_path = os.path.join(options["dest"], distribution.base_path) - to_export.append((repo_path, publication)) - except ObjectDoesNotExist: - logging.warning( - "No publication found for the repo published at '{}': skipping".format( - distribution.base_path + + def _collect_for_domain(domain, alias): + if options.get("distribution_path_prefix"): + distributions = Distribution.objects.using(alias).filter( + base_path__startswith=options["distribution_path_prefix"] + ) + else: + distributions = Distribution.objects.using(alias).all() + + if options["type"]: + distributions = distributions.filter(pulp_type__startswith=options["type"]) + + for distribution in distributions: + if distribution.publication: + publication = distribution.publication + elif distribution.repository: + repository = distribution.repository + try: + publication = ( + Publication.objects.using(alias) + .filter( + repository_version__in=repository.versions.all(), + complete=True, + ) + .latest("repository_version", "pulp_created") ) - ) + repo_path = os.path.join(options["dest"], distribution.base_path) + to_export.append((repo_path, publication)) + except ObjectDoesNotExist: + logging.warning( + "No publication found for the repo published at '{}' in " + "domain '{}': skipping".format(distribution.base_path, domain.name) + ) + + for_each_domain(_collect_for_domain) # Go through all the target directories first, if any of them are dirty, print warnings # and exit - unless the user explicitly asked to go through with it anyway. diff --git a/pulpcore/app/management/commands/handle-artifact-checksums.py b/pulpcore/app/management/commands/handle-artifact-checksums.py index 11c3ba72233..d8699561ea0 100644 --- a/pulpcore/app/management/commands/handle-artifact-checksums.py +++ b/pulpcore/app/management/commands/handle-artifact-checksums.py @@ -8,6 +8,7 @@ from pulpcore import constants from pulpcore.app import pulp_hashlib +from pulpcore.app.util import for_each_domain from pulpcore.plugin.models import ( Artifact, Content, @@ -51,19 +52,27 @@ def _print_out_repository_version_hrefs(self, repo_versions): ) ) - def _show_on_demand_content(self, checksums): + def _show_on_demand_content(self, checksums, alias): query = Q(pk__in=[]) for checksum in checksums: query |= Q(**{f"{checksum}__isnull": False}) - remote_artifacts = RemoteArtifact.objects.filter(query).filter( - content_artifact__artifact__isnull=True + remote_artifacts = ( + RemoteArtifact.objects.using(alias) + .filter(query) + .filter(content_artifact__artifact__isnull=True) ) ras_size = remote_artifacts.aggregate(Sum("size"))["size__sum"] - content_artifacts = ContentArtifact.objects.filter(remoteartifact__pk__in=remote_artifacts) - content = Content.objects.filter(contentartifact__pk__in=content_artifacts) - repo_versions = RepositoryVersion.objects.with_content(content).select_related("repository") + content_artifacts = ContentArtifact.objects.using(alias).filter( + remoteartifact__pk__in=remote_artifacts + ) + content = Content.objects.using(alias).filter(contentartifact__pk__in=content_artifacts) + repo_versions = ( + RepositoryVersion.objects.using(alias) + .with_content(content) + .select_related("repository") + ) self.stdout.write( "Found {} on-demand content units with forbidden checksums.".format(content.count()) @@ -77,7 +86,7 @@ def _show_on_demand_content(self, checksums): self.stdout.write(_("\nAffected repository versions with remote content:")) self._print_out_repository_version_hrefs(repo_versions) - def _show_immediate_content(self, forbidden_checksums): + def _show_immediate_content(self, forbidden_checksums, alias): allowed_checksums = set( constants.ALL_KNOWN_CONTENT_CHECKSUMS.symmetric_difference(forbidden_checksums) ) @@ -89,10 +98,14 @@ def _show_immediate_content(self, forbidden_checksums): for allowed_checksum in allowed_checksums: query_required |= Q(**{f"{allowed_checksum}__isnull": True}) - artifacts = Artifact.objects.filter(query_forbidden | query_required) - content_artifacts = ContentArtifact.objects.filter(artifact__in=artifacts) - content = Content.objects.filter(contentartifact__pk__in=content_artifacts) - repo_versions = RepositoryVersion.objects.with_content(content).select_related("repository") + artifacts = Artifact.objects.using(alias).filter(query_forbidden | query_required) + content_artifacts = ContentArtifact.objects.using(alias).filter(artifact__in=artifacts) + content = Content.objects.using(alias).filter(contentartifact__pk__in=content_artifacts) + repo_versions = ( + RepositoryVersion.objects.using(alias) + .with_content(content) + .select_related("repository") + ) self.stdout.write( "Found {} downloaded content units with forbidden or missing checksums.".format( @@ -110,11 +123,11 @@ def _show_immediate_content(self, forbidden_checksums): self.stdout.write(_("\nAffected repository versions with present content:")) self._print_out_repository_version_hrefs(repo_versions) - def _download_artifact(self, artifact, checksum, file_path): + def _download_artifact(self, artifact, checksum, file_path, alias): restored = False - for ca in artifact.content_memberships.all(): + for ca in artifact.content_memberships.using(alias).all(): if not restored: - for ra in ca.remoteartifact_set.all(): + for ra in ca.remoteartifact_set.using(alias).all(): remote = ra.remote.cast() if remote.policy == "immediate": self.stdout.write(_("Restoring missing file {}").format(file_path)) @@ -153,8 +166,12 @@ def _report(self, allowed_checksums): allowed_checksums ) - self._show_on_demand_content(forbidden_checksums) - self._show_immediate_content(forbidden_checksums) + def _report_for_domain(domain, alias): + self.stdout.write(_("\n=== Domain '{name}' ===").format(name=domain.name)) + self._show_on_demand_content(forbidden_checksums, alias) + self._show_immediate_content(forbidden_checksums, alias) + + for_each_domain(_report_for_domain) def handle(self, *args, **options): if options["report"]: @@ -167,30 +184,36 @@ def handle(self, *args, **options): log.setLevel(logging.ERROR) hrefs = set() - for checksum in settings.ALLOWED_CONTENT_CHECKSUMS: - params = {f"{checksum}__isnull": True} - artifacts_qs = Artifact.objects.filter(**params) - artifacts = [] - for a in artifacts_qs.iterator(): - hasher = pulp_hashlib.new(checksum) - try: - with a.file as fp: - for chunk in fp.chunks(CHUNK_SIZE): - hasher.update(chunk) - setattr(a, checksum, hasher.hexdigest()) - except FileNotFoundError: - file_path = os.path.join(settings.MEDIA_ROOT, a.file.name) - restored = self._download_artifact(a, checksum, file_path) - if not restored: - hrefs.add(file_path) - artifacts.append(a) - - if len(artifacts) >= 1000: - Artifact.objects.bulk_update(objs=artifacts, fields=[checksum], batch_size=1000) - artifacts.clear() - - if artifacts: - Artifact.objects.bulk_update(objs=artifacts, fields=[checksum]) + + def _populate_missing_for_domain(domain, alias): + for checksum in settings.ALLOWED_CONTENT_CHECKSUMS: + params = {f"{checksum}__isnull": True} + artifacts_qs = Artifact.objects.using(alias).filter(**params) + artifacts = [] + for a in artifacts_qs.iterator(): + hasher = pulp_hashlib.new(checksum) + try: + with a.file as fp: + for chunk in fp.chunks(CHUNK_SIZE): + hasher.update(chunk) + setattr(a, checksum, hasher.hexdigest()) + except FileNotFoundError: + file_path = os.path.join(settings.MEDIA_ROOT, a.file.name) + restored = self._download_artifact(a, checksum, file_path, alias) + if not restored: + hrefs.add(file_path) + artifacts.append(a) + + if len(artifacts) >= 1000: + Artifact.objects.using(alias).bulk_update( + objs=artifacts, fields=[checksum], batch_size=1000 + ) + artifacts.clear() + + if artifacts: + Artifact.objects.using(alias).bulk_update(objs=artifacts, fields=[checksum]) + + for_each_domain(_populate_missing_for_domain) if hrefs: raise CommandError( @@ -200,12 +223,20 @@ def handle(self, *args, **options): forbidden_checksums = set(constants.ALL_KNOWN_CONTENT_CHECKSUMS).difference( settings.ALLOWED_CONTENT_CHECKSUMS ) - for checksum in forbidden_checksums: - search_params = {f"{checksum}__isnull": False} - update_params = {f"{checksum}": None} - artifacts_qs = Artifact.objects.filter(**search_params) - if artifacts_qs.exists(): - self.stdout.write("Removing forbidden checksum {} from database".format(checksum)) - artifacts_qs.update(**update_params) + + def _remove_forbidden_for_domain(domain, alias): + for checksum in forbidden_checksums: + search_params = {f"{checksum}__isnull": False} + update_params = {f"{checksum}": None} + artifacts_qs = Artifact.objects.using(alias).filter(**search_params) + if artifacts_qs.exists(): + self.stdout.write( + "Removing forbidden checksum {} from database (domain '{}')".format( + checksum, domain.name + ) + ) + artifacts_qs.update(**update_params) + + for_each_domain(_remove_forbidden_for_domain) self.stdout.write(_("Finished aligning checksums with settings.ALLOWED_CONTENT_CHECKSUMS")) diff --git a/pulpcore/app/management/commands/migrate-all.py b/pulpcore/app/management/commands/migrate-all.py new file mode 100644 index 00000000000..bc44c0cd1c0 --- /dev/null +++ b/pulpcore/app/management/commands/migrate-all.py @@ -0,0 +1,129 @@ +from contextlib import contextmanager +from gettext import gettext as _ + +from django.conf import settings +from django.core.management import BaseCommand, CommandError, call_command +from django.db import connections +from django.utils.timezone import now + +from pulpcore.app.models import MigrationStatus +from pulpcore.constants import MIGRATION_ORCHESTRATOR_LOCK + +DOMAIN_TABLE_CHECKPOINT = ["core"] + + +@contextmanager +def _orchestrator_lock(): + with connections["default"].cursor() as cursor: + cursor.execute("SELECT pg_try_advisory_lock(%s)", [MIGRATION_ORCHESTRATOR_LOCK]) + (acquired,) = cursor.fetchone() + if not acquired: + raise CommandError( + _( + "Could not acquire the migration-orchestrator advisory lock. Another " + "'migrate-all' run is already in progress." + ) + ) + try: + yield + finally: + cursor.execute("SELECT pg_advisory_unlock(%s)", [MIGRATION_ORCHESTRATOR_LOCK]) + + +class Command(BaseCommand): + """ + Migrate every configured database alias, in the correct order. + + Runs Django's migrate command once for each alias in DATABASES, migrating the + default alias first so that satellite aliases can bootstrap against it. Safe to + re-run; already-migrated aliases are simply no-ops. + """ + + help = __doc__ + + def add_arguments(self, parser): + parser.add_argument( + "--target", + nargs=2, + metavar=("APP", "MIGRATION"), + help=_( + "Roll back every alias to this migration (Django 'app migration_name' syntax, " + "e.g. 'core 0154'). Without --target, migrates every alias to its latest " + "migration." + ), + ) + + def handle(self, *args, **options): + target = options.get("target") + aliases = [alias for alias in settings.DATABASES if alias != "default"] + if target: + ordered_aliases = aliases + ["default"] + else: + ordered_aliases = ["default"] + aliases + + with _orchestrator_lock(): + for alias in ordered_aliases: + if alias == "default": + self._migrate_one(alias, target) + if not target: + self._sync_domains() + elif target: + self._migrate_one(alias, target) + else: + self._migrate_satellite_forward(alias) + + def _migrate_satellite_forward(self, alias): + self._migrate_one(alias, DOMAIN_TABLE_CHECKPOINT, record_status=False) + self._sync_domains(alias=alias) + self._migrate_one(alias, None) + + def _migrate_one(self, alias, target, record_status=True): + self.stdout.write(_("Migrating database alias '{alias}'...").format(alias=alias)) + args = ["migrate", "--database", alias, "--noinput"] + if target: + args.extend(target) + try: + call_command(*args) + except Exception as e: + if record_status: + self._record_status(alias, "failed", error=str(e)) + raise CommandError( + _("Migration failed for database alias '{alias}': {error}").format( + alias=alias, error=e + ) + ) from e + else: + if record_status: + self._record_status(alias, "complete", completed_at=now()) + self.stdout.write( + self.style.SUCCESS(_("Database alias '{alias}' migrated.").format(alias=alias)) + ) + + def _record_status(self, alias, status, **defaults): + defaults.setdefault("error", None) + try: + MigrationStatus.objects.update_or_create( + database_alias=alias, defaults={"status": status, **defaults} + ) + except Exception: + self.stderr.write( + self.style.WARNING( + _("Could not record MigrationStatus for alias '{alias}'.").format(alias=alias) + ) + ) + + def _sync_domains(self, alias=None): + try: + if alias: + call_command("sync-domains", alias=alias) + else: + call_command("sync-domains") + except Exception: + self.stderr.write( + self.style.WARNING( + _( + "Domain sync to satellite '{alias}' failed; continuing with satellite " + "migrations. Run 'pulpcore-manager sync-domains' manually afterwards." + ).format(alias=alias or "*") + ) + ) diff --git a/pulpcore/app/management/commands/migrate.py b/pulpcore/app/management/commands/migrate.py new file mode 100644 index 00000000000..e5551677f33 --- /dev/null +++ b/pulpcore/app/management/commands/migrate.py @@ -0,0 +1,18 @@ +from django.core.management.commands.migrate import Command as _DjangoMigrateCommand +from django.db.utils import DEFAULT_DB_ALIAS + +from pulpcore.app.contexts import with_migration_alias + + +class Command(_DjangoMigrateCommand): + """ + Wrapper around Django's built-in migrate command that also records which + --database alias is currently being migrated, for the duration of the run. + """ + + help = _DjangoMigrateCommand.__doc__ + + def handle(self, *args, **options): + alias = options.get("database") or DEFAULT_DB_ALIAS + with with_migration_alias(alias): + return super().handle(*args, **options) diff --git a/pulpcore/app/management/commands/move-domain.py b/pulpcore/app/management/commands/move-domain.py new file mode 100644 index 00000000000..ade75477e4f --- /dev/null +++ b/pulpcore/app/management/commands/move-domain.py @@ -0,0 +1,265 @@ +from datetime import timedelta +from gettext import gettext as _ + +from django.conf import settings +from django.core.management import BaseCommand, CommandError +from django.db import connections +from django.db.migrations.executor import MigrationExecutor +from django.db.utils import OperationalError +from django.utils.timezone import now + +from pulpcore.app.domain_move import ( + DomainMoveError, + copy_domain_data, + domain_move_lock, + estimate_domain_size, + verify_domain_data, +) +from pulpcore.app.domain_sync import ensure_domain_on_alias +from pulpcore.app.models import Domain, DomainMove, Task +from pulpcore.constants import TASK_INCOMPLETE_STATES + +DEFAULT_MONITORING_DAYS = 7 + + +class Command(BaseCommand): + """ + Move a domain's data-plane objects to a different database alias. + + Sets the domain read-only for the duration of the copy, copies its data to the + target alias, verifies the copy, and then cuts over. Use 'cleanup-moved-domain' + afterwards to remove the stale data left on the original alias. + """ + + help = __doc__ + + def add_arguments(self, parser): + parser.add_argument("domain", help=_("Name of the domain to move.")) + parser.add_argument( + "--to", + required=True, + dest="to_alias", + help=_("Target DATABASES alias to move the domain's data to."), + ) + parser.add_argument( + "--strategy", + default="read-only", + choices=["read-only", "incremental"], + help=_( + "Movement strategy. Only 'read-only' (Strategy A) is implemented; 'incremental' " + "(Strategy B) is not yet implemented." + ), + ) + parser.add_argument( + "--skip-copy", + action="store_true", + help=_( + "Skip the data-copy step, assuming the domain's data was already copied to the " + "target alias out-of-band. Verification and cutover still run." + ), + ) + parser.add_argument( + "--monitoring-days", + type=int, + default=DEFAULT_MONITORING_DAYS, + help=_( + "Length, in days, of the post-move monitoring window recorded on the " + "DomainMove row. Default: %(default)s." + ), + ) + parser.add_argument( + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help=_("Do not prompt for confirmation before starting the move."), + ) + + def handle(self, *args, **options): + if options["strategy"] == "incremental": + raise CommandError( + _("Strategy B (incremental sync) is not implemented. Use --strategy read-only.") + ) + + to_alias = options["to_alias"] + if to_alias not in settings.DATABASES: + raise CommandError( + _("'{alias}' is not a configured DATABASES alias.").format(alias=to_alias) + ) + + try: + domain = Domain.objects.using("default").get(name=options["domain"]) + except Domain.DoesNotExist: + raise CommandError(_("No domain named '{name}' exists.").format(name=options["domain"])) + + self._validate_preconditions(domain, to_alias) + + size_report = estimate_domain_size(domain, domain.database_alias) + self._print_size_report(domain, size_report) + + if options["interactive"]: + confirm = input( + _( + "This will make domain '{name}' read-only for the duration of the copy " + "(may take a long time for large domains) and then move its data from " + "'{source}' to '{target}'. Continue? [y/N]: " + ).format(name=domain.name, source=domain.database_alias, target=to_alias) + ) + if confirm.strip().lower() not in ("y", "yes"): + self.stdout.write(_("Aborted.")) + return + + with domain_move_lock(): + self._move(domain, to_alias, options) + + def _validate_preconditions(self, domain, to_alias): + if domain.name == "default": + raise CommandError( + _( + "The 'default' domain can never be moved -- it is the only domain " + "guaranteed to exist on every deployment, and bootstrap/migration code " + "throughout pulpcore assumes it is reachable without any domain context." + ) + ) + if domain.moving: + raise CommandError( + _( + "Domain '{name}' already has moving=True -- either another move is in " + "progress (check for a concurrent 'move-domain' run) or a previous move " + "was interrupted. Resolve manually (inspect the latest DomainMove row for " + "this domain) before retrying." + ).format(name=domain.name) + ) + if domain.database_alias == to_alias: + raise CommandError( + _("Domain '{name}' is already on alias '{alias}'.").format( + name=domain.name, alias=to_alias + ) + ) + + self._validate_alias_ready(to_alias) + + incomplete = Task.objects.using("default").filter( + pulp_domain=domain, state__in=TASK_INCOMPLETE_STATES + ) + if incomplete.exists(): + raise CommandError( + _( + "Domain '{name}' has {n} in-flight task(s) (waiting/running/canceling). " + "Wait for them to finish (or cancel them) before moving this domain -- " + "read-only mode does not preempt already-dispatched work." + ).format(name=domain.name, n=incomplete.count()) + ) + + def _validate_alias_ready(self, alias): + try: + connections[alias].ensure_connection() + except OperationalError as e: + raise CommandError( + _("Target alias '{alias}' is not reachable: {error}").format(alias=alias, error=e) + ) from e + executor = MigrationExecutor(connections[alias]) + targets = executor.loader.graph.leaf_nodes() + if executor.migration_plan(targets): + raise CommandError( + _( + "Target alias '{alias}' has pending migrations. Run 'pulpcore-manager " + "migrate-all' first." + ).format(alias=alias) + ) + + def _print_size_report(self, domain, size_report): + self.stdout.write(_("Size estimate for domain '{name}':").format(name=domain.name)) + for row in size_report: + if row["row_count"] == 0: + continue + self.stdout.write( + " {model}: {rows} row(s) (table total size: {size} bytes)".format( + model=row["model"], rows=row["row_count"], size=row["table_total_size_bytes"] + ) + ) + + def _move(self, domain, to_alias, options): + from_alias = domain.database_alias + move = DomainMove.objects.using("default").create( + domain=domain, from_alias=from_alias, to_alias=to_alias, started_at=now() + ) + try: + domain.moving = True + domain.save(update_fields=["moving"]) + + ensure_domain_on_alias(domain, to_alias) + + if options["skip_copy"]: + self.stdout.write( + self.style.WARNING( + _( + "--skip-copy given: assuming data was already copied to '{alias}'." + ).format(alias=to_alias) + ) + ) + else: + self.stdout.write( + _("Copying data from '{source}' to '{target}'...").format( + source=from_alias, target=to_alias + ) + ) + copied = copy_domain_data(domain, from_alias, to_alias) + for label, count in copied.items(): + if count: + self.stdout.write(f" {label}: {count} row(s) copied") + + self.stdout.write(_("Verifying copied data...")) + mismatches = verify_domain_data(domain, from_alias, to_alias) + if mismatches: + for m in mismatches: + self.stderr.write( + self.style.ERROR( + " {model}: source={source_count} rows (checksum " + "{source_checksum}), target={target_count} rows (checksum " + "{target_checksum})".format(**m) + ) + ) + raise DomainMoveError( + "Verification failed for {n} model(s); see above. Domain '{name}' left " + "read-only on its original alias ('{alias}') -- no cutover performed. Fix " + "the discrepancy (e.g. re-run without --skip-copy) and retry.".format( + n=len(mismatches), name=domain.name, alias=from_alias + ) + ) + + domain.database_alias = to_alias + domain.moving = False + domain.save(update_fields=["database_alias", "moving"]) + + cutover_at = now() + move.cutover_at = cutover_at + move.monitoring_until = cutover_at + timedelta(days=options["monitoring_days"]) + move.status = "completed" + move.save(update_fields=["cutover_at", "monitoring_until", "status"]) + + self.stdout.write( + self.style.SUCCESS( + _( + "Domain '{name}' moved from '{source}' to '{target}'. Monitor until " + "{until} before running 'cleanup-moved-domain {name}' (DomainMove " + "{move_id})." + ).format( + name=domain.name, + source=from_alias, + target=to_alias, + until=move.monitoring_until, + move_id=move.pk, + ) + ) + ) + except Exception as e: + move.status = "failed" + move.error = str(e) + move.save(update_fields=["status", "error"]) + if domain.moving: + domain.moving = False + domain.save(update_fields=["moving"]) + if isinstance(e, DomainMoveError): + raise CommandError(str(e)) from e + raise diff --git a/pulpcore/app/management/commands/reconcile-cross-plane-references.py b/pulpcore/app/management/commands/reconcile-cross-plane-references.py new file mode 100644 index 00000000000..28d0c084a00 --- /dev/null +++ b/pulpcore/app/management/commands/reconcile-cross-plane-references.py @@ -0,0 +1,73 @@ +from gettext import gettext as _ + +from django.conf import settings +from django.core.management import BaseCommand + +from pulpcore.app.tasks.reconciliation import reconcile_cross_plane_references + + +class Command(BaseCommand): + """ + Sweep for orphaned cross-plane object references and optionally purge them. + + Reports rows whose referenced object can no longer be found on its recorded + database alias. Safe to run at any time, including on a single-database deployment. + """ + + help = __doc__ + + def add_arguments(self, parser): + parser.add_argument( + "--dry-run", + action="store_true", + help=_("Report orphans without purging any of them, regardless of --purge-after-days."), + ) + parser.add_argument( + "--grace-period-minutes", + type=int, + default=None, + help=_( + "Skip rows updated more recently than this many minutes ago. Defaults to " + "settings.CROSS_PLANE_RECONCILIATION_GRACE_MINUTES (currently {default})." + ).format(default=settings.CROSS_PLANE_RECONCILIATION_GRACE_MINUTES), + ) + parser.add_argument( + "--purge-after-days", + type=int, + default=None, + help=_( + "Delete confirmed-orphaned rows older than this many days. 0 disables purging " + "(the default). Defaults to settings.CROSS_PLANE_RECONCILIATION_PURGE_AFTER_DAYS " + "(currently {default})." + ).format(default=settings.CROSS_PLANE_RECONCILIATION_PURGE_AFTER_DAYS), + ) + + def handle(self, *args, **options): + report = reconcile_cross_plane_references( + grace_period_minutes=options["grace_period_minutes"], + purge_after_days=options["purge_after_days"], + dry_run=options["dry_run"], + ) + + self.stdout.write( + _("Checked {checked} cross-plane row(s); found {orphaned} orphan(s).").format( + checked=report["checked"], orphaned=report["orphaned"] + ) + ) + for orphan in report["orphans"]: + self.stdout.write( + self.style.WARNING( + " {model} pk={pk} (recorded alias='{alias}', age={age}d)".format( + model=orphan["model"], + pk=orphan["pk"], + alias=orphan["alias"], + age=orphan["age_days"], + ) + ) + ) + if report["purged"]: + self.stdout.write( + self.style.SUCCESS(_("Purged {n} orphan(s).").format(n=report["purged"])) + ) + if not report["orphaned"]: + self.stdout.write(self.style.SUCCESS(_("No orphans found."))) diff --git a/pulpcore/app/management/commands/remove-plugin.py b/pulpcore/app/management/commands/remove-plugin.py index 7531670b03e..f7d45367b9c 100644 --- a/pulpcore/app/management/commands/remove-plugin.py +++ b/pulpcore/app/management/commands/remove-plugin.py @@ -5,7 +5,7 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.management import BaseCommand, CommandError, call_command -from django.db import IntegrityError, connection +from django.db import IntegrityError, connections from django.db.migrations.exceptions import IrreversibleError from django.db.models.signals import post_migrate @@ -92,16 +92,21 @@ def _remove_plugin_data(self, app_label): In some cases, the order in which models are removed matters, e.g. FK is a part of uniqueness constraint. Try to remove such problematic models later. """ + for alias in settings.DATABASES: + self._remove_plugin_data_from_alias(app_label, alias) + self._remove_indirect_plugin_data(app_label) + def _remove_plugin_data_from_alias(self, app_label, alias): + self.stdout.write(_("Removing {} plugin data from alias '{}'...").format(app_label, alias)) models_to_delete = set(apps.all_models[app_label].values()) prev_model_count = len(models_to_delete) + 1 while models_to_delete and len(models_to_delete) < prev_model_count: # while there is something to delete and something is being deleted on each iteration removed_models = set() for model in models_to_delete: - self.stdout.write(_("Removing model: {}").format(model)) + self.stdout.write(_("Removing model: {} (alias '{}')").format(model, alias)) try: - model.objects.filter().delete() + model.objects.using(alias).filter().delete() except IntegrityError: continue else: @@ -114,18 +119,16 @@ def _remove_plugin_data(self, app_label): # Never-happen case raise CommandError( ( - "Data for the following models can't be removed: {}. Please contact plugin " - "maintainers." - ).format(list(models_to_delete)) + "Data for the following models can't be removed on alias '{}': {}. Please " + "contact plugin maintainers." + ).format(alias, list(models_to_delete)) ) - self._remove_indirect_plugin_data(app_label) - - def _drop_plugin_tables(self, app_label): + def _drop_plugin_tables(self, app_label, alias): """ Drop plugin table with raw SQL. """ - with connection.cursor() as cursor: + with connections[alias].cursor() as cursor: cursor.execute(DROP_PLUGIN_TABLES_QUERY.format(app_label=app_label)) def _unapply_migrations(self, app_label): @@ -148,13 +151,20 @@ def _unapply_migrations(self, app_label): if app_config.label == "core": post_migrate.disconnect(sender=app_config, dispatch_uid="delete_anon_identifier") - try: - call_command("migrate", app_label=app_label, migration_name="zero") - except (IrreversibleError, Exception): - # a plugin has irreversible migrations or some other problem, drop the tables and fake - # that migrations are unapplied. - self._drop_plugin_tables(app_label) - call_command("migrate", app_label=app_label, migration_name="zero", fake=True) + for alias in settings.DATABASES: + try: + call_command("migrate", app_label=app_label, migration_name="zero", database=alias) + except (IrreversibleError, Exception): + # a plugin has irreversible migrations or some other problem, drop the tables and + # fake that migrations are unapplied. + self._drop_plugin_tables(app_label, alias) + call_command( + "migrate", + app_label=app_label, + migration_name="zero", + fake=True, + database=alias, + ) def handle(self, *args, **options): plugin_name = options["plugin_name"] diff --git a/pulpcore/app/management/commands/repository-size.py b/pulpcore/app/management/commands/repository-size.py index a8ab13f7705..9bb12fc0df3 100644 --- a/pulpcore/app/management/commands/repository-size.py +++ b/pulpcore/app/management/commands/repository-size.py @@ -7,8 +7,8 @@ from django.conf import settings from django.core.management import BaseCommand, CommandError -from pulpcore.app.models import Repository -from pulpcore.app.util import extract_pk, get_url +from pulpcore.app.models import Domain, Repository +from pulpcore.app.util import extract_pk, for_each_domain, get_url def gather_repository_sizes(repositories, include_versions=False, include_on_demand=False): @@ -106,22 +106,49 @@ def add_arguments(self, parser): def handle(self, *args, **options): """Implement the command.""" - domain = options.get("domain") + domain_name = options.get("domain") repository_hrefs = options.get("repositories") - if domain and repository_hrefs: + if domain_name and repository_hrefs: raise CommandError(_("--domain and --repositories are mutually exclusive")) - repositories = Repository.objects.all() + report = [] if repository_hrefs: repos_ids = [extract_pk(r) for r in repository_hrefs] - repositories = repositories.filter(pk__in=repos_ids) - elif domain: - repositories = repositories.filter(pulp_domain__name=domain) - - report = gather_repository_sizes( - repositories, - include_versions=options["include_versions"], - include_on_demand=options["include_on_demand"], - ) + for alias in settings.DATABASES: + repositories = Repository.objects.using(alias).filter(pk__in=repos_ids) + report.extend( + gather_repository_sizes( + repositories, + include_versions=options["include_versions"], + include_on_demand=options["include_on_demand"], + ) + ) + elif domain_name: + try: + domain = Domain.objects.get(name=domain_name) + except Domain.DoesNotExist: + raise CommandError(_("Domain '{name}' does not exist.").format(name=domain_name)) + repositories = Repository.objects.using(domain.database_alias).filter( + pulp_domain=domain + ) + report = gather_repository_sizes( + repositories, + include_versions=options["include_versions"], + include_on_demand=options["include_on_demand"], + ) + else: + + def _gather_for_domain(domain, alias): + repositories = Repository.objects.using(alias).filter(pulp_domain=domain) + report.extend( + gather_repository_sizes( + repositories, + include_versions=options["include_versions"], + include_on_demand=options["include_on_demand"], + ) + ) + + for_each_domain(_gather_for_domain) + json.dump(report, sys.stdout, indent=4) print() diff --git a/pulpcore/app/management/commands/rotate-db-key.py b/pulpcore/app/management/commands/rotate-db-key.py index 6bbec206e10..086206586c9 100644 --- a/pulpcore/app/management/commands/rotate-db-key.py +++ b/pulpcore/app/management/commands/rotate-db-key.py @@ -2,8 +2,9 @@ from gettext import gettext as _ from django.apps import apps +from django.conf import settings from django.core.management import BaseCommand -from django.db import connection, transaction +from django.db import connections, transaction from pulpcore.app.models import MasterModel from pulpcore.app.models.fields import EncryptedJSONField, EncryptedTextField @@ -40,6 +41,11 @@ def add_arguments(self, parser): def handle(self, *args, **options): dry_run = options["dry_run"] + for alias in settings.DATABASES: + self._rotate_alias(alias, dry_run) + + def _rotate_alias(self, alias, dry_run): + print(_("Rotating encrypted fields on database alias '{alias}'.").format(alias=alias)) for model in apps.get_models(): if issubclass(model, MasterModel) and model._meta.master_model is None: # This is a master model, and we will handle all it's descendents. @@ -51,23 +57,23 @@ def handle(self, *args, **options): ] if field_names: print( - _("Updating {fields} on {model}.").format( - model=model.__name__, fields=",".join(field_names) + _("Updating {fields} on {model} (alias '{alias}').").format( + model=model.__name__, fields=",".join(field_names), alias=alias ) ) exclude_filters = {f"{field_name}": None for field_name in field_names} - qs = model.objects.exclude(**exclude_filters).only(*field_names) - with suppress(DryRun), transaction.atomic(): + qs = model.objects.using(alias).exclude(**exclude_filters).only(*field_names) + with suppress(DryRun), transaction.atomic(using=alias): batch = [] for item in qs.iterator(): batch.append(item) if len(batch) >= 1024: - model.objects.bulk_update(batch, field_names) + model.objects.using(alias).bulk_update(batch, field_names) batch = [] if batch: - model.objects.bulk_update(batch, field_names) + model.objects.using(alias).bulk_update(batch, field_names) batch = [] if dry_run: - with connection.cursor() as cursor: + with connections[alias].cursor() as cursor: cursor.execute("SET CONSTRAINTS ALL IMMEDIATE") raise DryRun() diff --git a/pulpcore/app/management/commands/sync-domains.py b/pulpcore/app/management/commands/sync-domains.py new file mode 100644 index 00000000000..12aa62ff457 --- /dev/null +++ b/pulpcore/app/management/commands/sync-domains.py @@ -0,0 +1,89 @@ +from gettext import gettext as _ + +from django.core.management import BaseCommand, CommandError + +from pulpcore.app.domain_sync import reconcile_domains_to_alias, satellite_aliases + + +class Command(BaseCommand): + """ + Reconcile the Domain table against every configured satellite database alias. + + Run this after provisioning a new satellite, after any bulk Domain changes, or + periodically as a reconciliation job, to fix up any satellites that missed updates. + """ + + help = __doc__ + + def add_arguments(self, parser): + parser.add_argument( + "--dry-run", + action="store_true", + help=_("Report drift without writing any changes."), + ) + parser.add_argument( + "--alias", + help=_("Only reconcile this one satellite alias, instead of every configured alias."), + ) + + def handle(self, *args, **options): + dry_run = options["dry_run"] + if options.get("alias"): + if options["alias"] not in satellite_aliases(): + raise CommandError( + _("'{alias}' is not a configured satellite alias.").format( + alias=options["alias"] + ) + ) + aliases = [options["alias"]] + else: + aliases = satellite_aliases() + if not aliases: + self.stdout.write( + self.style.WARNING( + _("Only one database alias is configured; nothing to reconcile.") + ) + ) + return + + any_drift = False + for alias in aliases: + self.stdout.write(_("Reconciling alias '{alias}'...").format(alias=alias)) + report = reconcile_domains_to_alias(alias, dry_run=dry_run) + missing, extra, stale = report["missing"], report["extra"], report["stale"] + + if not (missing or extra or stale): + self.stdout.write(_(" No drift detected.")) + continue + + any_drift = True + if missing: + self.stdout.write( + _(" {n} domain(s) missing on '{alias}': {ids}").format( + n=len(missing), alias=alias, ids=", ".join(str(i) for i in missing) + ) + ) + if extra: + self.stdout.write( + _(" {n} domain(s) exist only on '{alias}' (orphaned): {ids}").format( + n=len(extra), alias=alias, ids=", ".join(str(i) for i in extra) + ) + ) + if stale: + self.stdout.write( + _(" {n} domain(s) out of sync on '{alias}': {ids}").format( + n=len(stale), alias=alias, ids=", ".join(str(i) for i in stale) + ) + ) + + if not dry_run: + self.stdout.write( + self.style.SUCCESS(_(" Reconciled alias '{alias}'.").format(alias=alias)) + ) + + if dry_run and any_drift: + self.stdout.write( + self.style.WARNING(_("Dry run: no changes were written. Re-run without --dry-run.")) + ) + elif not any_drift: + self.stdout.write(self.style.SUCCESS(_("All satellite aliases are in sync."))) diff --git a/pulpcore/app/migrations/0101_add_domain.py b/pulpcore/app/migrations/0101_add_domain.py index ac5363050ca..71cd734c33e 100644 --- a/pulpcore/app/migrations/0101_add_domain.py +++ b/pulpcore/app/migrations/0101_add_domain.py @@ -7,7 +7,6 @@ import pulpcore.app.models.fields import uuid - DEFAULT_DELETE_TRIGGER = """ CREATE OR REPLACE FUNCTION protect_default() RETURNS TRIGGER as $protect_default$ BEGIN @@ -28,42 +27,52 @@ def create_default_domain(apps, schema_editor): - Domain = apps.get_model('core', 'Domain') + if schema_editor.connection.alias != "default": + return + Domain = apps.get_model("core", "Domain") try: - default_domain = Domain.objects.get(name="default") + default_domain = Domain.objects.using("default").get(name="default") except Domain.DoesNotExist: default_domain = Domain( name="default", storage_class=settings.STORAGES["default"]["BACKEND"] ) - default_domain.save(skip_hooks=True) + default_domain.save(using="default", skip_hooks=True) class Migration(migrations.Migration): - dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), + ("contenttypes", "0002_remove_content_type_name"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('core', '0100_upstreampulp'), + ("core", "0100_upstreampulp"), ] operations = [ migrations.CreateModel( - name='Domain', + name="Domain", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('name', models.SlugField(unique=True)), - ('description', models.TextField(null=True)), - ('storage_class', models.TextField()), - ('storage_settings', pulpcore.app.models.fields.EncryptedJSONField(default=dict)), - ('redirect_to_object_storage', models.BooleanField(default=True)), - ('hide_guarded_distributions', models.BooleanField(default=False)), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("name", models.SlugField(unique=True)), + ("description", models.TextField(null=True)), + ("storage_class", models.TextField()), + ("storage_settings", pulpcore.app.models.fields.EncryptedJSONField(default=dict)), + ("redirect_to_object_storage", models.BooleanField(default=True)), + ("hide_guarded_distributions", models.BooleanField(default=False)), ], options={ - 'permissions': [('manage_roles_domain', 'Can manage role assignments on domain')], + "permissions": [("manage_roles_domain", "Can manage role assignments on domain")], }, - bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model, pulpcore.app.models.access_policy.AutoAddObjPermsMixin), + bases=( + django_lifecycle.mixins.LifecycleModelMixin, + models.Model, + pulpcore.app.models.access_policy.AutoAddObjPermsMixin, + ), ), migrations.RunSQL(DEFAULT_DELETE_TRIGGER, reverse_sql=REMOVE_DEFAULT_DELETE_TRIGGER), migrations.RunPython(code=create_default_domain, reverse_code=migrations.RunPython.noop), diff --git a/pulpcore/app/migrations/0155_domain_database_alias_domain_moving.py b/pulpcore/app/migrations/0155_domain_database_alias_domain_moving.py new file mode 100644 index 00000000000..784a4dd35f2 --- /dev/null +++ b/pulpcore/app/migrations/0155_domain_database_alias_domain_moving.py @@ -0,0 +1,28 @@ +# Generated by Django 5.2.13 on 2026-07-09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0154_task_api_version"), + ] + + operations = [ + migrations.AddField( + model_name="domain", + name="database_alias", + field=models.SlugField( + default="default", + help_text="DATABASES alias where this domain's data-plane objects reside.", + ), + ), + migrations.AddField( + model_name="domain", + name="moving", + field=models.BooleanField( + default=False, + help_text="True while this domain's data is being moved between database aliases.", + ), + ), + ] diff --git a/pulpcore/app/migrations/0156_createdresource_content_object_domain_and_more.py b/pulpcore/app/migrations/0156_createdresource_content_object_domain_and_more.py new file mode 100644 index 00000000000..244a4fad0b9 --- /dev/null +++ b/pulpcore/app/migrations/0156_createdresource_content_object_domain_and_more.py @@ -0,0 +1,53 @@ +# Generated by Django 5.2.15 on 2026-07-09 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0155_domain_database_alias_domain_moving"), + ] + + operations = [ + migrations.AddField( + model_name="createdresource", + name="content_object_domain", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="core.domain", + ), + ), + migrations.AddField( + model_name="exportedresource", + name="content_object_domain", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="core.domain", + ), + ), + migrations.AddField( + model_name="grouprole", + name="content_object_domain", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="core.domain", + ), + ), + migrations.AddField( + model_name="userrole", + name="content_object_domain", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="core.domain", + ), + ), + ] diff --git a/pulpcore/app/migrations/0157_migrationstatus.py b/pulpcore/app/migrations/0157_migrationstatus.py new file mode 100644 index 00000000000..0824c72bd8f --- /dev/null +++ b/pulpcore/app/migrations/0157_migrationstatus.py @@ -0,0 +1,49 @@ +# Generated by Django 5.2.15 on 2026-07-09 + +import django_lifecycle.mixins +import pulpcore.app.models.base +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0156_createdresource_content_object_domain_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="MigrationStatus", + fields=[ + ( + "pulp_id", + models.UUIDField( + default=pulpcore.app.models.base.pulp_uuid, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("database_alias", models.TextField(unique=True)), + ( + "status", + models.TextField( + choices=[ + ("pending", "Pending"), + ("running", "Running"), + ("complete", "Complete"), + ("failed", "Failed"), + ], + default="pending", + ), + ), + ("completed_at", models.DateTimeField(null=True)), + ("error", models.TextField(null=True)), + ], + options={ + "verbose_name_plural": "migration statuses", + }, + bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model), + ), + ] diff --git a/pulpcore/app/migrations/0158_domainmove.py b/pulpcore/app/migrations/0158_domainmove.py new file mode 100644 index 00000000000..4fb4403e81b --- /dev/null +++ b/pulpcore/app/migrations/0158_domainmove.py @@ -0,0 +1,62 @@ +# Generated by Django 5.2.15 on 2026-07-09 17:21 + +import django.db.models.deletion +import django_lifecycle.mixins +import pulpcore.app.models.base +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0157_migrationstatus"), + ] + + operations = [ + migrations.CreateModel( + name="DomainMove", + fields=[ + ( + "pulp_id", + models.UUIDField( + default=pulpcore.app.models.base.pulp_uuid, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("from_alias", models.SlugField()), + ("to_alias", models.SlugField()), + ("started_at", models.DateTimeField()), + ("cutover_at", models.DateTimeField(null=True)), + ("monitoring_until", models.DateTimeField(null=True)), + ("cleaned_up_at", models.DateTimeField(null=True)), + ( + "status", + models.TextField( + choices=[ + ("in_progress", "In Progress"), + ("completed", "Completed"), + ("failed", "Failed"), + ("cleaned_up", "Cleaned Up"), + ], + default="in_progress", + ), + ), + ("error", models.TextField(null=True)), + ( + "domain", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="moves", + to="core.domain", + ), + ), + ], + options={ + "ordering": ["-pulp_created"], + }, + bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model), + ), + ] diff --git a/pulpcore/app/models/__init__.py b/pulpcore/app/models/__init__.py index a1bf1ca1147..87e48500482 100644 --- a/pulpcore/app/models/__init__.py +++ b/pulpcore/app/models/__init__.py @@ -84,6 +84,10 @@ from .analytics import SystemID +from .migration_status import MigrationStatus + +from .domain_move import DomainMove + from .upload import ( Upload, UploadChunk, @@ -161,6 +165,8 @@ "TaskGroup", "TaskSchedule", "SystemID", + "MigrationStatus", + "DomainMove", "Upload", "UploadChunk", "GroupProgressReport", diff --git a/pulpcore/app/models/content.py b/pulpcore/app/models/content.py index e73d4b480ed..42da25bd240 100644 --- a/pulpcore/app/models/content.py +++ b/pulpcore/app/models/content.py @@ -26,6 +26,7 @@ from pulpcore.app import pulp_hashlib from pulpcore.app.models import BaseModel, MasterModel, fields, storage +from pulpcore.app.queryset import CrossDBQuerySetMixin from pulpcore.app.util import get_domain_pk, gpg_verify from pulpcore.constants import ALL_KNOWN_CONTENT_CHECKSUMS from pulpcore.exceptions import ( @@ -97,7 +98,7 @@ def bulk_get_or_create(self, objs, batch_size=None): return objs -class BulkTouchQuerySet(models.QuerySet): +class BulkTouchQuerySet(CrossDBQuerySetMixin, models.QuerySet): """ A query set that provides ``touch()``. """ @@ -694,7 +695,7 @@ def sort_key(ca): return c_key, a_key -class RemoteArtifactQuerySet(models.QuerySet): +class RemoteArtifactQuerySet(CrossDBQuerySetMixin, models.QuerySet): """QuerySet that provides methods for querying RemoteArtifact.""" def acs(self): diff --git a/pulpcore/app/models/domain.py b/pulpcore/app/models/domain.py index aff0dc4b22e..4a2f004e448 100644 --- a/pulpcore/app/models/domain.py +++ b/pulpcore/app/models/domain.py @@ -1,7 +1,9 @@ +from django.conf import settings from django.contrib.postgres.fields import HStoreField +from django.core.exceptions import ValidationError from django.core.files.storage import default_storage from django.db import models -from django_lifecycle import BEFORE_DELETE, BEFORE_UPDATE, hook +from django_lifecycle import BEFORE_CREATE, BEFORE_DELETE, BEFORE_UPDATE, hook from pulpcore.app.models import AutoAddObjPermsMixin, BaseModel from pulpcore.exceptions import DomainProtectedError @@ -43,6 +45,14 @@ class Domain(BaseModel, AutoAddObjPermsMixin): # Pulp settings that are appropriate to be set on a "per domain" level redirect_to_object_storage = models.BooleanField(default=True) hide_guarded_distributions = models.BooleanField(default=False) + database_alias = models.SlugField( + default="default", + help_text="DATABASES alias where this domain's data-plane objects reside.", + ) + moving = models.BooleanField( + default=False, + help_text="True while this domain's data is being moved between database aliases.", + ) def get_storage(self): """Returns this domain's instantiated storage class.""" @@ -65,13 +75,29 @@ def get_storage(self): def prevent_default_deletion(self): raise models.ProtectedError("Default domain can not be updated/deleted.", [self]) + @hook(BEFORE_CREATE) + @hook(BEFORE_UPDATE, when="database_alias", has_changed=True) + def _validate_database_alias(self): + if self.database_alias not in settings.DATABASES: + raise ValidationError( + { + "database_alias": ( + f"'{self.database_alias}' is not a configured DATABASES alias." + ) + } + ) + @hook(BEFORE_DELETE, when="name", is_not="default") def _cleanup_orphans_pre_delete(self): - protected_content_set = self.content_set.exclude(version_memberships__isnull=True) + protected_content_set = self.content_set.using(self.database_alias).exclude( + version_memberships__isnull=True + ) if protected_content_set.exists(): raise DomainProtectedError() - self.content_set.filter(version_memberships__isnull=True).delete() - for artifact in self.artifact_set.all().iterator(): + self.content_set.using(self.database_alias).filter( + version_memberships__isnull=True + ).delete() + for artifact in self.artifact_set.using(self.database_alias).all().iterator(): # Delete on by one to properly cleanup the storage. artifact.delete() diff --git a/pulpcore/app/models/domain_move.py b/pulpcore/app/models/domain_move.py new file mode 100644 index 00000000000..f39d9ddc7e7 --- /dev/null +++ b/pulpcore/app/models/domain_move.py @@ -0,0 +1,25 @@ +from django.db import models + +from pulpcore.app.models import BaseModel + +DOMAIN_MOVE_STATUS_CHOICES = ( + ("in_progress", "In Progress"), + ("completed", "Completed"), + ("failed", "Failed"), + ("cleaned_up", "Cleaned Up"), +) + + +class DomainMove(BaseModel): + domain = models.ForeignKey("Domain", on_delete=models.CASCADE, related_name="moves") + from_alias = models.SlugField() + to_alias = models.SlugField() + started_at = models.DateTimeField() + cutover_at = models.DateTimeField(null=True) + monitoring_until = models.DateTimeField(null=True) + cleaned_up_at = models.DateTimeField(null=True) + status = models.TextField(choices=DOMAIN_MOVE_STATUS_CHOICES, default="in_progress") + error = models.TextField(null=True) + + class Meta: + ordering = ["-pulp_created"] diff --git a/pulpcore/app/models/generic.py b/pulpcore/app/models/generic.py index 2d8420d4a01..1f4c055c235 100644 --- a/pulpcore/app/models/generic.py +++ b/pulpcore/app/models/generic.py @@ -5,14 +5,111 @@ https://docs.djangoproject.com/en/3.2/ref/contrib/contenttypes/#generic-relations """ +import logging + from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ObjectDoesNotExist from django.db import models from pulpcore.app.models.base import BaseModel +_logger = logging.getLogger(__name__) + + +_UNSET = object() + +_DOMAIN_WALK_MAX_DEPTH = 2 + + +def _resolve_domain_id(value, _depth=0, _seen=None): + domain_id = getattr(value, "pulp_domain_id", None) + if domain_id is not None: + return domain_id + if _depth >= _DOMAIN_WALK_MAX_DEPTH: + return None + if _seen is None: + _seen = set() + if value.pk is not None: + key = (type(value), value.pk) + if key in _seen: + return None + _seen.add(key) + for field in value._meta.get_fields(): + if not (field.many_to_one or field.one_to_one) or not getattr(field, "concrete", False): + continue + try: + related = getattr(value, field.name) + except ObjectDoesNotExist: + continue + if related is None or not hasattr(related, "_meta"): + continue + resolved = _resolve_domain_id(related, _depth + 1, _seen) + if resolved is not None: + return resolved + return None + + +class DomainResolvedGenericRelation: + def __init__(self, *args, **kwargs): + has_content_object = "content_object" in kwargs + content_object = kwargs.pop("content_object", None) + super().__init__(*args, **kwargs) + if has_content_object: + self.content_object = content_object + + @property + def content_object(self): + cached = self.__dict__.get("_content_object_cache", _UNSET) + if cached is not _UNSET: + return cached + if self.content_type_id is None or self.object_id is None: + return None + model_class = self.content_type.model_class() + if self.content_object_domain_id is not None: + alias = self.content_object_domain.database_alias + try: + resolved = model_class.objects.using(alias).get(pk=self.object_id) + except model_class.DoesNotExist: + _logger.warning( + "content_object for %s (pk=%s) not found on alias '%s' " + "(content_type_id=%s, object_id=%s). The referenced object may have been " + "deleted, or Domain replication for this row's domain may be stale -- run " + "'pulpcore-manager sync-domains' to check.", + self._meta.label, + self.pk, + alias, + self.content_type_id, + self.object_id, + ) + resolved = None + else: + try: + resolved = model_class._base_manager.using(self._state.db or "default").get( + pk=self.object_id + ) + except model_class.DoesNotExist: + resolved = None + self.__dict__["_content_object_cache"] = resolved + return resolved + + @content_object.setter + def content_object(self, value): + self.__dict__["_content_object_cache"] = value + if value is None: + self.content_type = None + self.object_id = None + self.content_object_domain_id = None + return + gfk = type(self)._content_object + self.content_type = ContentType.objects.db_manager("default").get_for_model( + value, for_concrete_model=gfk.for_concrete_model + ) + self.object_id = value.pk + self.content_object_domain_id = _resolve_domain_id(value) + -class GenericRelationModel(BaseModel): +class GenericRelationModel(DomainResolvedGenericRelation, BaseModel): """Base model class for implementing Generic Relations. This class provides the required fields to implement generic relations. Instances of @@ -22,7 +119,10 @@ class GenericRelationModel(BaseModel): content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.UUIDField() - content_object = GenericForeignKey("content_type", "object_id", for_concrete_model=False) + _content_object = GenericForeignKey("content_type", "object_id", for_concrete_model=False) + content_object_domain = models.ForeignKey( + "core.Domain", null=True, on_delete=models.SET_NULL, related_name="+" + ) class Meta: abstract = True diff --git a/pulpcore/app/models/migration_status.py b/pulpcore/app/models/migration_status.py new file mode 100644 index 00000000000..8ce3d4c1b3e --- /dev/null +++ b/pulpcore/app/models/migration_status.py @@ -0,0 +1,20 @@ +from django.db import models + +from pulpcore.app.models import BaseModel + +MIGRATION_STATUS_CHOICES = ( + ("pending", "Pending"), + ("running", "Running"), + ("complete", "Complete"), + ("failed", "Failed"), +) + + +class MigrationStatus(BaseModel): + database_alias = models.TextField(unique=True) + status = models.TextField(choices=MIGRATION_STATUS_CHOICES, default="pending") + completed_at = models.DateTimeField(null=True) + error = models.TextField(null=True) + + class Meta: + verbose_name_plural = "migration statuses" diff --git a/pulpcore/app/models/publication.py b/pulpcore/app/models/publication.py index b953be30db9..9b5d0ba6b4a 100644 --- a/pulpcore/app/models/publication.py +++ b/pulpcore/app/models/publication.py @@ -21,6 +21,7 @@ from pulpcore.app.files import PulpTemporaryUploadedFile from pulpcore.app.models import AutoAddObjPermsMixin +from pulpcore.app.queryset import CrossDBQuerySetMixin from pulpcore.app.util import cache_key, get_domain_pk, get_url, retain_distributed_pub_enabled from pulpcore.cache import Cache from pulpcore.responses import ArtifactResponse @@ -33,7 +34,7 @@ _logger = logging.getLogger(__name__) -class PublicationQuerySet(models.QuerySet): +class PublicationQuerySet(CrossDBQuerySetMixin, models.QuerySet): """A queryset that provides publication filtering methods.""" def with_content(self, content): diff --git a/pulpcore/app/models/repository.py b/pulpcore/app/models/repository.py index 5be034f5ffa..b917cb25bc1 100644 --- a/pulpcore/app/models/repository.py +++ b/pulpcore/app/models/repository.py @@ -18,6 +18,7 @@ from django_lifecycle import AFTER_UPDATE, BEFORE_CREATE, BEFORE_DELETE, hook from rest_framework.exceptions import APIException +from pulpcore.app.queryset import CrossDBQuerySetMixin from pulpcore.app.util import ( batch_qs, cache_key, @@ -882,7 +883,7 @@ class Meta: ) -class RepositoryVersionQuerySet(models.QuerySet): +class RepositoryVersionQuerySet(CrossDBQuerySetMixin, models.QuerySet): """A queryset that provides repository version filtering methods.""" def complete(self): diff --git a/pulpcore/app/models/role.py b/pulpcore/app/models/role.py index c7db1d3fa24..7835b531e0b 100644 --- a/pulpcore/app/models/role.py +++ b/pulpcore/app/models/role.py @@ -5,6 +5,7 @@ from django.db import models from pulpcore.app.models import BaseModel, Group +from pulpcore.app.models.generic import DomainResolvedGenericRelation class Role(BaseModel): @@ -27,7 +28,7 @@ class Role(BaseModel): permissions = models.ManyToManyField(Permission) -class UserRole(BaseModel): +class UserRole(DomainResolvedGenericRelation, BaseModel): """ Join table for user to role associations with optional content object. @@ -46,7 +47,10 @@ class UserRole(BaseModel): role = models.ForeignKey(Role, related_name="object_users", on_delete=models.CASCADE) content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, null=True) object_id = models.TextField(null=True) - content_object = GenericForeignKey("content_type", "object_id", for_concrete_model=False) + _content_object = GenericForeignKey("content_type", "object_id", for_concrete_model=False) + content_object_domain = models.ForeignKey( + "Domain", null=True, on_delete=models.SET_NULL, related_name="+" + ) domain = models.ForeignKey("Domain", null=True, on_delete=models.CASCADE) class Meta: @@ -57,7 +61,7 @@ class Meta: ] -class GroupRole(BaseModel): +class GroupRole(DomainResolvedGenericRelation, BaseModel): """ Join table for group to role associations with optional content object. @@ -74,7 +78,10 @@ class GroupRole(BaseModel): role = models.ForeignKey(Role, related_name="object_groups", on_delete=models.CASCADE) content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, null=True) object_id = models.TextField(null=True) - content_object = GenericForeignKey("content_type", "object_id", for_concrete_model=False) + _content_object = GenericForeignKey("content_type", "object_id", for_concrete_model=False) + content_object_domain = models.ForeignKey( + "Domain", null=True, on_delete=models.SET_NULL, related_name="+" + ) domain = models.ForeignKey("Domain", null=True, on_delete=models.CASCADE) class Meta: diff --git a/pulpcore/app/queryset.py b/pulpcore/app/queryset.py new file mode 100644 index 00000000000..3c066bec370 --- /dev/null +++ b/pulpcore/app/queryset.py @@ -0,0 +1,37 @@ +from django.db import models +from django.db.models import Q + + +class CrossDBQuerySetMixin: + def filter(self, *args, **kwargs): + from pulpcore.app.db_router import is_multi_db_routing_active + + if not is_multi_db_routing_active(): + return super().filter(*args, **kwargs) + args = tuple(self._resolve_cross_db_q(a) if isinstance(a, Q) else a for a in args) + self._resolve_cross_db_kwargs(kwargs) + return super().filter(*args, **kwargs) + + def exclude(self, *args, **kwargs): + from pulpcore.app.db_router import is_multi_db_routing_active + + if not is_multi_db_routing_active(): + return super().exclude(*args, **kwargs) + args = tuple(self._resolve_cross_db_q(a) if isinstance(a, Q) else a for a in args) + self._resolve_cross_db_kwargs(kwargs) + return super().exclude(*args, **kwargs) + + def _resolve_cross_db_kwargs(self, kwargs): + for key, value in list(kwargs.items()): + if isinstance(value, models.QuerySet) and value.db != self.db: + kwargs[key] = list(value) + + def _resolve_cross_db_q(self, q): + for i, child in enumerate(q.children): + if isinstance(child, Q): + self._resolve_cross_db_q(child) + elif isinstance(child, tuple): + key, value = child + if isinstance(value, models.QuerySet) and value.db != self.db: + q.children[i] = (key, list(value)) + return q diff --git a/pulpcore/app/role_util.py b/pulpcore/app/role_util.py index 103c68541e3..a5d49426c9c 100644 --- a/pulpcore/app/role_util.py +++ b/pulpcore/app/role_util.py @@ -132,9 +132,11 @@ def get_objects_for_user_roles( ): return qs - user_role_pks = user.object_roles.filter( - domain__isnull=True, role__permissions=permission - ).values_list("object_id", flat=True) + user_role_pks = list( + user.object_roles.filter(domain__isnull=True, role__permissions=permission).values_list( + "object_id", flat=True + ) + ) final_q = Q(pk_str__in=user_role_pks) if accept_domain_perms and hasattr(qs.model, "pulp_domain"): domains = list( @@ -155,9 +157,11 @@ def get_objects_for_user_roles( ) if use_groups: - group_role_pks = GroupRole.objects.filter( - group__in=user.groups.all(), role__permissions=permission, domain__isnull=True - ).values_list("object_id", flat=True) + group_role_pks = list( + GroupRole.objects.filter( + group__in=user.groups.all(), role__permissions=permission, domain__isnull=True + ).values_list("object_id", flat=True) + ) final_q |= Q(pk_str__in=group_role_pks) return qs.annotate(pk_str=Cast("pk", output_field=CharField())).filter(final_q) @@ -575,3 +579,21 @@ def get_groups_with_perms( for_concrete_model=for_concrete_model, ) return qs.distinct() + + +def cleanup_roles_for_deleted_object(instance): + content_type = ContentType.objects.get_for_model(instance, for_concrete_model=False) + object_id = str(instance.pk) + UserRole.objects.using("default").filter( + content_type=content_type, object_id=object_id + ).delete() + GroupRole.objects.using("default").filter( + content_type=content_type, object_id=object_id + ).delete() + + +def on_any_model_post_delete(sender, instance, **kwargs): + from pulpcore.app.models import BaseModel + + if isinstance(instance, BaseModel): + cleanup_roles_for_deleted_object(instance) diff --git a/pulpcore/app/serializers/status.py b/pulpcore/app/serializers/status.py index 7863b5435ae..e30df5d762f 100644 --- a/pulpcore/app/serializers/status.py +++ b/pulpcore/app/serializers/status.py @@ -47,6 +47,22 @@ class DatabaseConnectionSerializer(serializers.Serializer): ) +class DatabaseStatusSerializer(serializers.Serializer): + alias = serializers.CharField(help_text=_("The settings.DATABASES alias this entry reports on")) + + connected = serializers.BooleanField( + help_text=_("Info about whether the app can connect to this database alias") + ) + + migrations_complete = serializers.BooleanField( + help_text=_( + "Whether this database alias has no pending migrations. Null if connectivity " + "could not be established, since migration status can't be determined in that case." + ), + allow_null=True, + ) + + class RedisConnectionSerializer(serializers.Serializer): """ Serializer for information about the Redis connection @@ -126,6 +142,14 @@ class StatusSerializer(serializers.Serializer): help_text=_("Database connection information") ) + databases = DatabaseStatusSerializer( + help_text=_( + "Per-alias connectivity and migration-completeness status for every configured " + "settings.DATABASES alias (one entry per alias, including 'default')." + ), + many=True, + ) + redis_connection = RedisConnectionSerializer( required=False, help_text=_("Redis connection information"), diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py index d34aebf8a17..cd7c50abb84 100644 --- a/pulpcore/app/settings.py +++ b/pulpcore/app/settings.py @@ -310,6 +310,12 @@ TASK_PROTECTION_TIME = 0 TMPFILE_PROTECTION_TIME = 0 +CROSS_PLANE_RECONCILIATION_GRACE_MINUTES = 60 + +CROSS_PLANE_RECONCILIATION_PURGE_AFTER_DAYS = 0 + +CROSS_PLANE_RECONCILIATION_INTERVAL_MINUTES = 24 * 60 + REMOTE_USER_ENVIRON_NAME = "REMOTE_USER" REMOTE_USER_OPENAPI_SECURITY_SCHEME = {"type": "mutualTLS"} diff --git a/pulpcore/app/tasks/reconciliation.py b/pulpcore/app/tasks/reconciliation.py new file mode 100644 index 00000000000..650f12007a2 --- /dev/null +++ b/pulpcore/app/tasks/reconciliation.py @@ -0,0 +1,98 @@ +from datetime import timedelta +from logging import getLogger + +from django.conf import settings +from django.utils import timezone + +from pulpcore.app.models import CreatedResource, ExportedResource +from pulpcore.app.models.role import GroupRole, UserRole + +log = getLogger(__name__) + +_GFK_MODELS = (CreatedResource, ExportedResource, UserRole, GroupRole) + + +def _candidate_rows(model, cutoff): + return model.objects.using("default").filter( + content_object_domain_id__isnull=False, + pulp_last_updated__lt=cutoff, + ) + + +def _target_exists(row): + model_class = row.content_type.model_class() + alias = row.content_object_domain.database_alias + return model_class.objects.using(alias).filter(pk=row.object_id).exists() + + +def reconcile_cross_plane_references( + grace_period_minutes=None, + purge_after_days=None, + dry_run=False, +): + if grace_period_minutes is None: + grace_period_minutes = settings.CROSS_PLANE_RECONCILIATION_GRACE_MINUTES + if purge_after_days is None: + purge_after_days = settings.CROSS_PLANE_RECONCILIATION_PURGE_AFTER_DAYS + + cutoff = timezone.now() - timedelta(minutes=grace_period_minutes) + purge_cutoff = timezone.now() - timedelta(days=purge_after_days) if purge_after_days else None + + report = {"checked": 0, "orphaned": 0, "purged": 0, "orphans": []} + + for model in _GFK_MODELS: + qs = _candidate_rows(model, cutoff) + for row in qs.iterator(): + report["checked"] += 1 + if not _target_exists(row): + alias = row.content_object_domain.database_alias + age = timezone.now() - row.pulp_last_updated + log.error( + "content_object for %s (pk=%s) not found on alias '%s' " + "(content_type_id=%s, object_id=%s). The referenced object may have been " + "deleted, or Domain replication for this row's domain may be stale -- run " + "'pulpcore-manager sync-domains' to check.", + model._meta.label, + row.pk, + alias, + row.content_type_id, + row.object_id, + ) + report["orphaned"] += 1 + report["orphans"].append( + { + "model": model._meta.label, + "pk": str(row.pk), + "alias": alias, + "age_days": age.days, + } + ) + if not dry_run and purge_cutoff and row.pulp_last_updated < purge_cutoff: + log.warning( + "Purging orphaned cross-plane row %s (pk=%s, alias=%s, age=%sd) -- " + "unresolvable content_object older than " + "CROSS_PLANE_RECONCILIATION_PURGE_AFTER_DAYS=%s.", + model._meta.label, + row.pk, + alias, + age.days, + purge_after_days, + ) + row.delete() + report["purged"] += 1 + + if report["orphaned"]: + log.error( + "Cross-plane reconciliation found %s orphaned reference(s) out of %s checked " + "(%s purged). See preceding log lines for details on each. Run " + "'pulpcore-manager reconcile-cross-plane-references' for a full report.", + report["orphaned"], + report["checked"], + report["purged"], + ) + else: + log.info( + "Cross-plane reconciliation checked %s row(s), found no orphans.", report["checked"] + ) + + return report diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py index 76f2a5b47fb..397d61928de 100644 --- a/pulpcore/app/util.py +++ b/pulpcore/app/util.py @@ -2,7 +2,7 @@ import os import socket import zlib -from contextlib import ExitStack +from contextlib import ExitStack, contextmanager from datetime import timedelta from functools import lru_cache from gettext import gettext as _ @@ -15,14 +15,19 @@ from django.apps import apps from django.conf import settings -from django.db import connection +from django.db import connections from django.db.models import Model, UUIDField from rest_framework.reverse import reverse as drf_reverse from rest_framework.serializers import ValidationError from pulpcore.app import models from pulpcore.app.apps import pulp_plugin_configs -from pulpcore.app.contexts import _current_domain, _current_user_func, current_pulp_api_version +from pulpcore.app.contexts import ( + _current_domain, + _current_user_func, + current_pulp_api_version, + with_domain, +) from pulpcore.app.loggers import deprecation_logger from pulpcore.exceptions.validation import InvalidSignatureError @@ -508,6 +513,11 @@ def configure_cleanup(): ), ("tasks", "pulpcore.app.tasks.purge.purge", settings.TASK_PROTECTION_TIME), ("content", "pulpcore.app.tasks.orphan.orphan_cleanup", settings.ORPHAN_PROTECTION_TIME), + ( + "cross-plane references", + "pulpcore.app.tasks.reconciliation.reconcile_cross_plane_references", + settings.CROSS_PLANE_RECONCILIATION_INTERVAL_MINUTES, + ), ]: if protection_time > 0: dispatch_interval = timedelta(minutes=protection_time) @@ -646,7 +656,7 @@ def get_domain_pk(): if default_domain: return default_domain.pk # If we haven't cached the default_domain then use raw SQL to get its PK - with connection.cursor() as cursor: + with connections["default"].cursor() as cursor: cursor.execute("SELECT pulp_id FROM core_domain WHERE name = 'default'") row = cursor.fetchone() return row[0] @@ -658,6 +668,18 @@ def set_domain(new_domain): return new_domain +@contextmanager +def domain_db(domain): + with with_domain(domain): + yield domain.database_alias + + +def for_each_domain(callback): + for domain in models.Domain.objects.all(): + with domain_db(domain) as alias: + callback(domain, alias) + + def cache_key(base_path): """Returns the base-key(s) used in the Cache for the passed base_path(s).""" if settings.DOMAIN_ENABLED: diff --git a/pulpcore/app/views/status.py b/pulpcore/app/views/status.py index 6ac28912c1c..d751e95789d 100644 --- a/pulpcore/app/views/status.py +++ b/pulpcore/app/views/status.py @@ -4,6 +4,9 @@ from gettext import gettext as _ from django.conf import settings +from django.db import connections +from django.db import utils as db_utils +from django.db.migrations.executor import MigrationExecutor from django.db.models import Sum from drf_spectacular.utils import extend_schema from rest_framework.response import Response @@ -78,6 +81,7 @@ def get(self, request, **kwargs): redis_status = {"connected": False} db_status = {"connected": self._get_db_conn_status()} + databases = self._get_databases_status() online_workers = AppStatus.objects.online().filter(app_type="worker") online_api_apps = AppStatus.objects.online().filter(app_type="api") @@ -94,6 +98,7 @@ def get(self, request, **kwargs): "online_api_apps": online_api_apps, "online_content_apps": online_content_apps, "database_connection": db_status, + "databases": databases, "redis_connection": redis_status, "storage": _disk_usage(), "content_settings": content_settings, @@ -129,6 +134,33 @@ def _get_db_conn_status(): else: return True + @staticmethod + def _get_databases_status(): + results = [] + for alias in settings.DATABASES: + connected = False + migrations_complete = None + try: + connections[alias].ensure_connection() + connected = True + executor = MigrationExecutor(connections[alias]) + targets = executor.loader.graph.leaf_nodes() + migrations_complete = not executor.migration_plan(targets) + except db_utils.OperationalError: + _logger.exception( + _("Cannot connect to database alias '%(alias)s' during status check."), + {"alias": alias}, + ) + except Exception: + _logger.exception( + _("Failed to determine migration status for database alias '%(alias)s'."), + {"alias": alias}, + ) + results.append( + {"alias": alias, "connected": connected, "migrations_complete": migrations_complete} + ) + return results + @staticmethod def _get_redis_conn_status(): """ diff --git a/pulpcore/app/viewsets/task.py b/pulpcore/app/viewsets/task.py index 3dba074e95f..81e45aab62a 100644 --- a/pulpcore/app/viewsets/task.py +++ b/pulpcore/app/viewsets/task.py @@ -12,6 +12,7 @@ from pulpcore.app.models import ( AppStatus, + Artifact, CreatedResource, ProfileArtifact, RepositoryVersion, @@ -290,8 +291,10 @@ def profile_artifacts(self, request, pk, **kwargs): task = self.get_object() data = {} - for pa in ProfileArtifact.objects.select_related("artifact").filter(task=task): - data[pa.name] = get_artifact_url(pa.artifact) + alias = task.pulp_domain.database_alias + for pa in ProfileArtifact.objects.filter(task=task): + artifact = Artifact.objects.using(alias).get(pk=pa.artifact_id) + data[pa.name] = get_artifact_url(artifact) return Response({"urls": data}) diff --git a/pulpcore/constants.py b/pulpcore/constants.py index 9b48a5f91e1..f7a72c86dbf 100644 --- a/pulpcore/constants.py +++ b/pulpcore/constants.py @@ -9,6 +9,8 @@ TASK_UNBLOCKING_LOCK = 84 TASK_METRICS_LOCK = 74 WORKER_CLEANUP_LOCK = 11 +MIGRATION_ORCHESTRATOR_LOCK = 137 +DOMAIN_MOVE_LOCK = 138 # Reasons to send along a task worker wakeup call. TASK_WAKEUP_UNBLOCK = "unblock" diff --git a/pulpcore/middleware.py b/pulpcore/middleware.py index fd06ed63a60..bdadd34e499 100644 --- a/pulpcore/middleware.py +++ b/pulpcore/middleware.py @@ -1,9 +1,13 @@ import re import time +from gettext import gettext as _ from os import environ from django.conf import settings from django.core.exceptions import MiddlewareNotUsed +from django.db import connections +from django.db.utils import Error as DjangoDBError +from django.http import JsonResponse from django.http.response import Http404 from pulpcore.app.contexts import current_pulp_api_version, x_task_diagnostics_var @@ -16,6 +20,8 @@ ) from pulpcore.metrics import init_otel_meter +_SAFE_HTTP_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) + class DomainMiddleware: """ @@ -47,10 +53,40 @@ def process_view(self, request, view_func, view_args, view_kwargs): domain = Domain.objects.get(name=domain_name) except Domain.DoesNotExist: raise Http404() + degraded_response = self._degraded_response(request, domain) + if degraded_response is not None: + return degraded_response set_domain(domain) setattr(request, "pulp_domain", domain) return None + @staticmethod + def _degraded_response(request, domain): + alias = domain.database_alias + if alias != "default": + try: + connections[alias].ensure_connection() + except DjangoDBError: + return JsonResponse( + { + "detail": _( + "Database for domain '{name}' is currently unavailable." + ).format(name=domain.name) + }, + status=503, + ) + if domain.moving and request.method not in _SAFE_HTTP_METHODS: + return JsonResponse( + { + "detail": _( + "Domain '{name}' is currently being moved to a different database; " + "write operations are temporarily unavailable." + ).format(name=domain.name) + }, + status=503, + ) + return None + class APIRootRewriteMiddleware: """ diff --git a/pulpcore/tasking/redis_worker.py b/pulpcore/tasking/redis_worker.py index 59ffdca9f95..fbd85b1b407 100644 --- a/pulpcore/tasking/redis_worker.py +++ b/pulpcore/tasking/redis_worker.py @@ -20,7 +20,7 @@ import redis from django.conf import settings -from django.db import DatabaseError, IntegrityError, connection, transaction +from django.db import DatabaseError, IntegrityError, connection, connections, transaction from django.utils import timezone from pulpcore.app.apps import pulp_plugin_configs @@ -472,6 +472,29 @@ def is_compatible(self, task): return False return True + def is_domain_available(self, task): + domain = task.pulp_domain + if domain.moving: + _logger.info( + _("Domain '%s' is being moved between databases; deferring task %s."), + domain.name, + task.pk, + ) + return False + alias = domain.database_alias + if alias != "default": + try: + connections[alias].ensure_connection() + except DatabaseError: + _logger.warning( + _("Database alias '%s' for domain '%s' is unavailable; deferring task %s."), + alias, + domain.name, + task.pk, + ) + return False + return True + def fetch_task(self): """ Fetch an available waiting task using Redis locks. @@ -708,8 +731,7 @@ def handle_tasks(self): # No task found break - if not self.is_compatible(task): - # Incompatible task, add to ignored list + if not self.is_compatible(task) or not self.is_domain_available(task): self.ignored_task_ids.append(task.pk) # Atomically release task lock + resource locks so other workers can attempt it self._maybe_release_locks(task, mark_released=False) diff --git a/pulpcore/tasking/worker.py b/pulpcore/tasking/worker.py index eae0d248cc7..3abc611f9aa 100644 --- a/pulpcore/tasking/worker.py +++ b/pulpcore/tasking/worker.py @@ -10,7 +10,7 @@ from tempfile import TemporaryDirectory from django.conf import settings -from django.db import DatabaseError, IntegrityError, connection, transaction +from django.db import DatabaseError, IntegrityError, connection, connections, transaction from django.db.models import Case, Count, F, Max, Value, When from django.utils import timezone from packaging.version import parse as parse_version @@ -328,6 +328,29 @@ def is_compatible(self, task): return False return True + def is_domain_available(self, task): + domain = task.pulp_domain + if domain.moving: + _logger.info( + _("Domain '%s' is being moved between databases; deferring task %s."), + domain.name, + task.pk, + ) + return False + alias = domain.database_alias + if alias != "default": + try: + connections[alias].ensure_connection() + except DatabaseError: + _logger.warning( + _("Database alias '%s' for domain '%s' is unavailable; deferring task %s."), + alias, + domain.name, + task.pk, + ) + return False + return True + def unblock_tasks(self): """Iterate over waiting tasks and mark them unblocked accordingly. @@ -605,7 +628,11 @@ def handle_unblocked_tasks(self): elif task.state == TASK_STATES.RUNNING: # A running task without a lock must be abandoned. self.cancel_abandoned_task(task, TASK_STATES.FAILED, "Worker has gone missing.") - elif task.state == TASK_STATES.WAITING and self.is_compatible(task): + elif ( + task.state == TASK_STATES.WAITING + and self.is_compatible(task) + and self.is_domain_available(task) + ): if task.immediate: self.supervise_immediate_task(task) else: diff --git a/pulpcore/tests/functional/api/test_tasking.py b/pulpcore/tests/functional/api/test_tasking.py index d9e95a56a0c..fce14d1c14c 100644 --- a/pulpcore/tests/functional/api/test_tasking.py +++ b/pulpcore/tests/functional/api/test_tasking.py @@ -13,7 +13,7 @@ from pulpcore.client.pulpcore import ApiException from pulpcore.constants import IMMEDIATE_TIMEOUT -from pulpcore.tests.functional.utils import PulpTaskError, download_file +from pulpcore.tests.functional.utils import SLEEP_TIME, PulpTaskError, download_file @pytest.fixture(scope="module") @@ -476,10 +476,24 @@ def test_finalizer_task_runs_after_all_siblings(dispatch_task_group, monitor_tas @pytest.mark.parallel def test_cancel_task_group(pulpcore_bindings, dispatch_task_group, gen_user): """Test that task groups can be canceled.""" + cancel_retry_timeout = 60 + + def _cancel_task_group_retrying(): + deadline = time.monotonic() + cancel_retry_timeout + while True: + try: + return pulpcore_bindings.TaskGroupsApi.task_groups_cancel( + tgroup_href, {"state": "canceled"} + ) + except ApiException as e: + if e.status != 409 or time.monotonic() >= deadline: + raise + time.sleep(SLEEP_TIME) + kwargs = {"inbetween": 1, "intervals": [10, 10, 10, 10, 10]} tgroup_href = dispatch_task_group("pulpcore.app.tasks.test.dummy_group_task", kwargs=kwargs) - tgroup = pulpcore_bindings.TaskGroupsApi.task_groups_cancel(tgroup_href, {"state": "canceled"}) + tgroup = _cancel_task_group_retrying() for task in tgroup.tasks: assert task.state in ["canceled", "canceling"] @@ -497,7 +511,7 @@ def test_cancel_task_group(pulpcore_bindings, dispatch_task_group, gen_user): assert "You do not have permission" in e.value.message with gen_user(model_roles=["core.task_owner"]): - pulpcore_bindings.TaskGroupsApi.task_groups_cancel(tgroup_href, {"state": "canceled"}) + _cancel_task_group_retrying() LT_TIMEOUT = IMMEDIATE_TIMEOUT / 2 diff --git a/pulpcore/tests/unit/content/test_handler.py b/pulpcore/tests/unit/content/test_handler.py index 045efdad6cd..f29f10c2a87 100644 --- a/pulpcore/tests/unit/content/test_handler.py +++ b/pulpcore/tests/unit/content/test_handler.py @@ -8,7 +8,8 @@ from django.db import IntegrityError from django_guid import clear_guid, set_guid -from pulpcore.app.models import AppStatus +from pulpcore.app.contexts import with_domain +from pulpcore.app.models import AppStatus, Domain from pulpcore.constants import TASK_STATES from pulpcore.content.handler import CheckpointListings, Handler, PathNotResolved from pulpcore.plugin.models import ( @@ -22,6 +23,7 @@ Repository, RepositoryVersion, ) +from pulpcore.tests.unit.test_multi_database_routing import SATELLITE_ALIAS, requires_multi_db @pytest.fixture @@ -318,6 +320,49 @@ def test_pull_through_save_single_artifact_content( assert ra is not None +@requires_multi_db +@pytest.mark.django_db(databases=["default", SATELLITE_ALIAS]) +def test_pull_through_save_single_artifact_content_multi_db( + request123, download_result_mock, monkeypatch, tmp_path +): + domain = Domain.objects.create( + name="ki27-pull-through-domain", + storage_class="pulpcore.app.models.storage.FileSystem", + storage_settings={"location": str(tmp_path)}, + database_alias=SATELLITE_ALIAS, + ) + try: + with with_domain(domain): + remote = Remote.objects.create(name="123", url="https://123") + handler = Handler() + remote.get_remote_artifact_content_type = Mock(return_value=Content) + content_init_mock = Mock(return_value=Content()) + monkeypatch.setattr(Content, "init_from_artifact_and_relative_path", content_init_mock) + ca = ContentArtifact(relative_path="c123") + ra = RemoteArtifact(url=f"{remote.url}/c123", remote=remote, content_artifact=ca) + + content_artifacts = handler._save_artifact(download_result_mock, ra, request=request123) + artifact = content_artifacts[ra.content_artifact.relative_path].artifact + + assert Artifact.objects.using(SATELLITE_ALIAS).filter(pk=artifact.pk).exists() + assert not Artifact.objects.using("default").filter(pk=artifact.pk).exists() + saved_ra = ( + RemoteArtifact.objects.using(SATELLITE_ALIAS) + .filter(url=f"{remote.url}/c123", remote=remote) + .first() + ) + assert saved_ra is not None + assert saved_ra.pulp_domain_id == domain.pk + finally: + for alias in {SATELLITE_ALIAS, "default"}: + RemoteArtifact.objects.using(alias).filter(pulp_domain=domain).delete() + ContentArtifact.objects.using(alias).filter(content__pulp_domain=domain).delete() + Content.objects.using(alias).filter(pulp_domain=domain).delete() + Artifact.objects.using(alias).filter(pulp_domain=domain).delete() + Remote.objects.using(alias).filter(pulp_domain=domain).delete() + domain.delete() + + def test_pull_through_save_multi_artifact_content( remote123, request123, download_result_mock, monkeypatch, tmp_path ): diff --git a/pulpcore/tests/unit/models/test_generic.py b/pulpcore/tests/unit/models/test_generic.py new file mode 100644 index 00000000000..402bc248471 --- /dev/null +++ b/pulpcore/tests/unit/models/test_generic.py @@ -0,0 +1,65 @@ +from uuid import uuid4 + +import pytest + +from pulpcore.app.contexts import with_task_context +from pulpcore.app.models import CreatedResource, RepositoryVersion, Task +from pulpcore.app.models.generic import _resolve_domain_id + +from pulp_file.app.models import FileRepository + + +@pytest.fixture +def task(): + t = Task.objects.create(name="test-generic-relation-task") + yield t + t.delete() + + +@pytest.mark.django_db +def test_content_object_returns_none_for_deleted_domain_scoped_target(task): + with with_task_context(task): + repository = FileRepository.objects.create(name=str(uuid4())) + created_resource = CreatedResource.objects.create(content_object=repository) + assert created_resource.content_object_domain_id is not None + + repository.delete() + + created_resource = CreatedResource.objects.get(pk=created_resource.pk) + assert created_resource.content_object is None + + +@pytest.mark.django_db +def test_content_object_resolves_existing_domain_scoped_target(task): + with with_task_context(task): + repository = FileRepository.objects.create(name=str(uuid4())) + created_resource = CreatedResource.objects.create(content_object=repository) + + created_resource = CreatedResource.objects.get(pk=created_resource.pk) + resolved = created_resource.content_object + assert resolved is not None + assert resolved.pk == repository.pk + + +@pytest.mark.django_db +def test_resolve_domain_id_walks_transitive_fk(task): + with with_task_context(task): + repository = FileRepository.objects.create(name=str(uuid4())) + version = RepositoryVersion.objects.create(repository=repository, number=1) + + assert getattr(version, "pulp_domain_id", None) is None + assert _resolve_domain_id(version) == repository.pulp_domain_id + + +@pytest.mark.django_db +def test_content_object_domain_id_set_for_repository_version(task): + with with_task_context(task): + repository = FileRepository.objects.create(name=str(uuid4())) + version = RepositoryVersion.objects.create(repository=repository, number=1) + created_resource = CreatedResource.objects.create(content_object=version) + assert created_resource.content_object_domain_id == repository.pulp_domain_id + + created_resource = CreatedResource.objects.get(pk=created_resource.pk) + resolved = created_resource.content_object + assert resolved is not None + assert resolved.pk == version.pk diff --git a/pulpcore/tests/unit/models/test_remote.py b/pulpcore/tests/unit/models/test_remote.py index bb394816d37..cdf23d0b562 100644 --- a/pulpcore/tests/unit/models/test_remote.py +++ b/pulpcore/tests/unit/models/test_remote.py @@ -1,16 +1,21 @@ +from pathlib import Path from uuid import uuid4 import pytest from cryptography.fernet import InvalidToken +from django.conf import settings from django.core.management import call_command from django.db import connection +from pulpcore.app.contexts import with_domain from pulpcore.app.models import Domain, Remote from pulpcore.app.models.fields import EncryptedTextField, _fernet TEST_KEY1 = b"hPCIFQV/upbvPRsEpgS7W32XdFA2EQgXnMtyNAekebQ=" TEST_KEY2 = b"6Xyv+QezAQ+4R870F5qsgKcngzmm46caDB2gyo9qnpc=" +SATELLITE_ALIAS = "data_1" + @pytest.fixture def fake_fernet(tmp_path, settings): @@ -50,27 +55,61 @@ def test_encrypted_proxy_password(fake_fernet): assert proxy_password == "test" -@pytest.mark.django_db +@pytest.mark.django_db(databases=list(settings.DATABASES)) def test_rotate_db_key(fake_fernet): remote = Remote.objects.create(name=uuid4(), proxy_password="test") domain = Domain.objects.create(name=uuid4(), storage_settings={"base_path": "/foo"}) - next(fake_fernet) # new + old key - - call_command("rotate-db-key") - - next(fake_fernet) # new key - - del remote.proxy_password - assert remote.proxy_password == "test" - del domain.storage_settings - assert domain.storage_settings == {"base_path": "/foo"} - - next(fake_fernet) # old key - - del remote.proxy_password - with pytest.raises(InvalidToken): - remote.proxy_password - del domain.storage_settings - with pytest.raises(InvalidToken): - domain.storage_settings + satellite_remote = None + satellite_domain = None + if SATELLITE_ALIAS in settings.DATABASES: + satellite_domain = Domain.objects.create( + name=uuid4(), + storage_class="pulpcore.app.models.storage.FileSystem", + storage_settings={"base_path": "/satellite"}, + database_alias=SATELLITE_ALIAS, + ) + with with_domain(satellite_domain): + satellite_remote = Remote.objects.create( + name=uuid4(), proxy_password="satellite-secret" + ) + assert not Remote.objects.using("default").filter(pk=satellite_remote.pk).exists() + assert Remote.objects.using(SATELLITE_ALIAS).filter(pk=satellite_remote.pk).exists() + + try: + next(fake_fernet) # new + old key + + call_command("rotate-db-key") + + next(fake_fernet) # new key + + del remote.proxy_password + assert remote.proxy_password == "test" + del domain.storage_settings + assert domain.storage_settings == {"base_path": "/foo"} + + if satellite_remote is not None: + satellite_remote = Remote.objects.using(SATELLITE_ALIAS).get(pk=satellite_remote.pk) + assert satellite_remote.proxy_password == "satellite-secret" + + next(fake_fernet) # old key + + del remote.proxy_password + with pytest.raises(InvalidToken): + remote.proxy_password + del domain.storage_settings + with pytest.raises(InvalidToken): + domain.storage_settings + + if satellite_remote is not None: + with pytest.raises(InvalidToken): + Remote.objects.using(SATELLITE_ALIAS).get(pk=satellite_remote.pk) + finally: + if satellite_remote is not None or satellite_domain is not None: + key_file = Path(settings.DB_ENCRYPTION_KEY) + key_file.write_bytes(TEST_KEY2 + b"\n" + TEST_KEY1) + _fernet.cache_clear() + if satellite_remote is not None: + Remote.objects.using(SATELLITE_ALIAS).filter(pk=satellite_remote.pk).delete() + if satellite_domain is not None: + satellite_domain.delete() diff --git a/pulpcore/tests/unit/test_db_router.py b/pulpcore/tests/unit/test_db_router.py new file mode 100644 index 00000000000..f47a388789e --- /dev/null +++ b/pulpcore/tests/unit/test_db_router.py @@ -0,0 +1,48 @@ +import pytest +from django.db import router as django_router + +from pulpcore.app.db_router import PulpDomainRouter, _database_alias, is_multi_db_routing_active +from pulpcore.app.models import Domain + + +@pytest.mark.django_db +def test_database_alias_reads_loaded_field(): + domain = Domain.objects.get(name="default") + assert _database_alias(domain) == "default" + + +@pytest.mark.django_db +def test_database_alias_does_not_query_when_field_is_deferred(django_assert_num_queries): + domain = Domain.objects.only("pk", "name").get(name="default") + assert "database_alias" not in domain.__dict__, ( + "test setup assumption broken: .only('pk', 'name') should defer 'database_alias'" + ) + with django_assert_num_queries(0): + assert _database_alias(domain) == "default" + + +@pytest.mark.django_db +def test_database_alias_reads_non_default_value_when_loaded(): + domain = Domain.objects.get(name="default") + domain.__dict__["database_alias"] = "data_1" + assert _database_alias(domain) == "data_1" + + +def test_is_multi_db_routing_active_false_by_default(): + original_routers = django_router.routers + try: + django_router.routers = [] + assert is_multi_db_routing_active() is False + finally: + django_router.routers = original_routers + + +def test_is_multi_db_routing_active_true_when_registered_then_false_after(): + original_routers = django_router.routers + try: + django_router.routers = [] + assert is_multi_db_routing_active() is False + django_router.routers = [PulpDomainRouter()] + assert is_multi_db_routing_active() is True + finally: + django_router.routers = original_routers diff --git a/pulpcore/tests/unit/test_domain_move.py b/pulpcore/tests/unit/test_domain_move.py new file mode 100644 index 00000000000..93065299a7d --- /dev/null +++ b/pulpcore/tests/unit/test_domain_move.py @@ -0,0 +1,117 @@ +import hashlib + +import pytest +from django.core.files.base import ContentFile +from django.core.management import call_command + +from pulpcore.app.contexts import with_domain +from pulpcore.app.models import ( + Artifact, + ContentArtifact, + Domain, + DomainMove, +) + +from pulp_file.app.models import FileContent, FileRepository + +from .test_multi_database_routing import SATELLITE_ALIAS, requires_multi_db + +pytestmark = [requires_multi_db, pytest.mark.django_db(databases=["default", SATELLITE_ALIAS])] + + +@pytest.fixture +def hot_domain(tmp_path): + domain = Domain.objects.create( + name="move-test-domain", + storage_class="pulpcore.app.models.storage.FileSystem", + storage_settings={"location": str(tmp_path)}, + ) + with with_domain(domain): + repo = FileRepository.objects.create(name="move-test-repo", pulp_domain=domain) + data = b"move-domain integration test content" + digests = { + alg: hashlib.new(alg, data).hexdigest() + for alg in ("sha224", "sha256", "sha384", "sha512") + } + artifact = Artifact.objects.create( + file=ContentFile(data, name="x"), size=len(data), pulp_domain=domain, **digests + ) + content = FileContent.objects.create( + relative_path="a/b.txt", digest=digests["sha256"], pulp_domain=domain + ) + ContentArtifact.objects.create(artifact=artifact, content=content, relative_path="a/b.txt") + version = repo.new_version() + with version: + version.add_content(FileContent.objects.filter(pk=content.pk)) + yield domain + domain.refresh_from_db() + for alias in {domain.database_alias, "default"}: + FileRepository.objects.using(alias).filter(pulp_domain=domain).delete() + FileContent.objects.using(alias).filter(pulp_domain=domain).delete() + Artifact.objects.using(alias).filter(pulp_domain=domain).delete() + domain.delete() + + +class TestMoveDomain: + def test_move_domain_relocates_data_and_verifies_cleanly(self, hot_domain): + call_command("move-domain", hot_domain.name, "--to", SATELLITE_ALIAS, "--noinput") + + hot_domain.refresh_from_db() + assert hot_domain.database_alias == SATELLITE_ALIAS + assert hot_domain.moving is False + + assert FileRepository.objects.using(SATELLITE_ALIAS).filter(pulp_domain=hot_domain).exists() + assert Artifact.objects.using(SATELLITE_ALIAS).filter(pulp_domain=hot_domain).exists() + assert FileRepository.objects.using("default").filter(pulp_domain=hot_domain).exists() + + move = DomainMove.objects.using("default").get(domain=hot_domain, status="completed") + assert move.from_alias == "default" + assert move.to_alias == SATELLITE_ALIAS + assert move.cutover_at is not None + assert move.monitoring_until is not None + + def test_move_domain_refuses_default_domain(self): + default_domain = Domain.objects.using("default").get(name="default") + with pytest.raises(Exception, match="default"): + call_command("move-domain", default_domain.name, "--to", SATELLITE_ALIAS, "--noinput") + + def test_move_domain_refuses_unconfigured_alias(self, hot_domain): + with pytest.raises(Exception, match="not a configured"): + call_command("move-domain", hot_domain.name, "--to", "not-a-real-alias", "--noinput") + + def test_move_domain_refuses_same_alias(self, hot_domain): + with pytest.raises(Exception, match="already on alias"): + call_command( + "move-domain", hot_domain.name, "--to", hot_domain.database_alias, "--noinput" + ) + + +class TestCleanupMovedDomain: + def test_cleanup_requires_force(self, hot_domain): + call_command("move-domain", hot_domain.name, "--to", SATELLITE_ALIAS, "--noinput") + with pytest.raises(Exception, match="--force"): + call_command("cleanup-moved-domain", hot_domain.name) + + def test_cleanup_deletes_stale_source_copy(self, hot_domain): + call_command("move-domain", hot_domain.name, "--to", SATELLITE_ALIAS, "--noinput") + call_command("cleanup-moved-domain", hot_domain.name, "--force") + + hot_domain.refresh_from_db() + assert not FileRepository.objects.using("default").filter(pulp_domain=hot_domain).exists() + assert not Artifact.objects.using("default").filter(pulp_domain=hot_domain).exists() + assert FileRepository.objects.using(SATELLITE_ALIAS).filter(pulp_domain=hot_domain).exists() + + move = DomainMove.objects.using("default").get(domain=hot_domain, status="cleaned_up") + assert move.cleaned_up_at is not None + + def test_cleanup_refuses_domain_still_on_default(self, hot_domain): + with pytest.raises(Exception, match="nothing to clean up"): + call_command("cleanup-moved-domain", hot_domain.name, "--force") + + +class TestDomainSize: + def test_domain_size_reports_row_counts(self, hot_domain, capsys): + call_command("domain-size", hot_domain.name) + out = capsys.readouterr().out + assert "file.FileRepository" in out + assert "core.Artifact" in out diff --git a/pulpcore/tests/unit/test_domain_sync.py b/pulpcore/tests/unit/test_domain_sync.py new file mode 100644 index 00000000000..53720a22f12 --- /dev/null +++ b/pulpcore/tests/unit/test_domain_sync.py @@ -0,0 +1,115 @@ +import pytest + +from pulpcore.app.domain_sync import ( + ensure_domain_on_alias, + reconcile_domains_to_alias, + replicate_domain_delete, + replicate_domain_save, +) +from pulpcore.app.models import Domain + +from .test_multi_database_routing import SATELLITE_ALIAS, requires_multi_db + +pytestmark = [requires_multi_db, pytest.mark.django_db(databases=["default", SATELLITE_ALIAS])] + + +@pytest.fixture +def default_hosted_domain(): + domain = Domain.objects.create( + name="sync-test-default-hosted", + storage_class="pulpcore.app.models.storage.FileSystem", + storage_settings={"location": "/tmp/sync-test-default-hosted"}, + ) + yield domain + domain.delete() + + +@pytest.fixture +def satellite_hosted_domain(): + domain = Domain.objects.create( + name="sync-test-satellite-hosted", + storage_class="pulpcore.app.models.storage.FileSystem", + storage_settings={"location": "/tmp/sync-test-satellite-hosted"}, + database_alias=SATELLITE_ALIAS, + ) + yield domain + domain.delete() + + +class TestReplicationScoping: + def test_default_domain_replicates_to_satellite(self): + default_domain = Domain.objects.using("default").get(name="default") + + replicate_domain_save(default_domain) + + assert Domain.objects.using(SATELLITE_ALIAS).filter(pk=default_domain.pk).exists() + + def test_domain_hosted_on_default_is_not_replicated_to_satellite(self, default_hosted_domain): + assert ( + not Domain.objects.using(SATELLITE_ALIAS).filter(pk=default_hosted_domain.pk).exists() + ) + + def test_domain_hosted_on_satellite_is_replicated_only_there(self, satellite_hosted_domain): + assert Domain.objects.using(SATELLITE_ALIAS).filter(pk=satellite_hosted_domain.pk).exists() + + def test_replicate_domain_delete_only_targets_current_alias(self, satellite_hosted_domain): + assert Domain.objects.using(SATELLITE_ALIAS).filter(pk=satellite_hosted_domain.pk).exists() + + replicate_domain_delete(satellite_hosted_domain) + + assert ( + not Domain.objects.using(SATELLITE_ALIAS).filter(pk=satellite_hosted_domain.pk).exists() + ) + + +class TestEnsureDomainOnAlias: + def test_seeds_row_on_alias_regardless_of_current_database_alias(self, default_hosted_domain): + assert ( + not Domain.objects.using(SATELLITE_ALIAS).filter(pk=default_hosted_domain.pk).exists() + ) + + ensure_domain_on_alias(default_hosted_domain, SATELLITE_ALIAS) + + assert Domain.objects.using(SATELLITE_ALIAS).filter(pk=default_hosted_domain.pk).exists() + replicated = Domain.objects.using(SATELLITE_ALIAS).get(pk=default_hosted_domain.pk) + assert replicated.database_alias == "default" + + +class TestReconcileDomainsToAlias: + def test_domain_hosted_elsewhere_is_not_flagged_missing(self, default_hosted_domain): + report = reconcile_domains_to_alias(SATELLITE_ALIAS, dry_run=True) + + assert default_hosted_domain.pulp_id not in report["missing"] + assert ( + not Domain.objects.using(SATELLITE_ALIAS).filter(pk=default_hosted_domain.pk).exists() + ) + + def test_domain_hosted_here_but_missing_is_reconciled(self, satellite_hosted_domain): + Domain.objects.using(SATELLITE_ALIAS).filter(pk=satellite_hosted_domain.pk).delete() + + report = reconcile_domains_to_alias(SATELLITE_ALIAS) + + assert satellite_hosted_domain.pulp_id in report["missing"] + assert Domain.objects.using(SATELLITE_ALIAS).filter(pk=satellite_hosted_domain.pk).exists() + + def test_domain_moved_away_is_pruned_as_extra(self, satellite_hosted_domain): + assert Domain.objects.using(SATELLITE_ALIAS).filter(pk=satellite_hosted_domain.pk).exists() + + satellite_hosted_domain.database_alias = "default" + satellite_hosted_domain.save(update_fields=["database_alias"]) + + report = reconcile_domains_to_alias(SATELLITE_ALIAS) + + assert satellite_hosted_domain.pulp_id in report["extra"] + assert ( + not Domain.objects.using(SATELLITE_ALIAS).filter(pk=satellite_hosted_domain.pk).exists() + ) + + def test_dry_run_reports_extra_without_deleting(self, satellite_hosted_domain): + satellite_hosted_domain.database_alias = "default" + satellite_hosted_domain.save(update_fields=["database_alias"]) + + report = reconcile_domains_to_alias(SATELLITE_ALIAS, dry_run=True) + + assert satellite_hosted_domain.pulp_id in report["extra"] + assert Domain.objects.using(SATELLITE_ALIAS).filter(pk=satellite_hosted_domain.pk).exists() diff --git a/pulpcore/tests/unit/test_middleware.py b/pulpcore/tests/unit/test_middleware.py index dbb2649eedd..86410c2af5b 100644 --- a/pulpcore/tests/unit/test_middleware.py +++ b/pulpcore/tests/unit/test_middleware.py @@ -29,6 +29,7 @@ def test_does_db_lookup_when_flag_not_set(self, mock_set_domain, mock_domain_obj view_class = type("NormalView", (), {}) view_func = MagicMock(view_class=view_class) view_kwargs = {"pulp_domain": "default"} + mock_domain_objects.get.return_value = MagicMock(database_alias="default", moving=False) self.middleware.process_view(self.request, view_func, [], view_kwargs) diff --git a/pulpcore/tests/unit/test_multi_database_routing.py b/pulpcore/tests/unit/test_multi_database_routing.py new file mode 100644 index 00000000000..ed7d20dea3d --- /dev/null +++ b/pulpcore/tests/unit/test_multi_database_routing.py @@ -0,0 +1,227 @@ +from contextlib import contextmanager +from unittest import mock + +import pytest +from django.conf import settings +from django.core.management import call_command +from django.db.utils import OperationalError + +from pulpcore.app.contexts import with_domain +from pulpcore.app.db_router import is_multi_db_routing_active +from pulpcore.app.models import ( + ContentArtifact, + Domain, + MigrationStatus, + Remote, + RemoteArtifact, + Repository, + Task, +) +from pulpcore.constants import TASK_STATES + +SATELLITE_ALIAS = "data_1" + +requires_multi_db = pytest.mark.skipif( + SATELLITE_ALIAS not in settings.DATABASES or not is_multi_db_routing_active(), + reason=( + f"Multi-database routing tests require a '{SATELLITE_ALIAS}' alias in settings.DATABASES " + f"(set PULP_DATABASES__{SATELLITE_ALIAS}__* env vars to a second real Postgres instance) " + "and PulpDomainRouter registered in DATABASE_ROUTERS." + ), +) + +pytestmark = [requires_multi_db, pytest.mark.django_db(databases=["default", SATELLITE_ALIAS])] + + +@contextmanager +def _satellite_domain(**extra_fields): + domain = Domain.objects.create( + name=f"test-satellite-domain-{extra_fields.get('_suffix', '')}".rstrip("-"), + storage_class="pulpcore.app.models.storage.FileSystem", + database_alias=SATELLITE_ALIAS, + **{k: v for k, v in extra_fields.items() if k != "_suffix"}, + ) + try: + yield domain + finally: + domain.delete() + + +class TestMigrateAll: + def test_migrate_all_migrates_every_alias(self): + call_command("migrate-all") + + statuses = {m.database_alias: m.status for m in MigrationStatus.objects.all()} + for alias in settings.DATABASES: + assert statuses.get(alias) == "complete", ( + f"Expected MigrationStatus for alias '{alias}' to be 'complete', got " + f"{statuses.get(alias)!r}" + ) + + def test_migrate_all_reconciles_domain_table_to_satellite(self): + call_command("migrate-all") + + default_domain = Domain.objects.using("default").get(name="default") + assert Domain.objects.using(SATELLITE_ALIAS).filter(pk=default_domain.pk).exists(), ( + "The 'default' Domain row should have been replicated onto the satellite alias by " + "migrate-all's Domain-sync step." + ) + + +class TestPulpDomainRouter: + def test_data_plane_object_routes_to_satellite_alias(self): + with _satellite_domain(_suffix="routing") as domain: + with with_domain(domain): + repo = Repository.objects.create(name=f"{domain.name}-repo", pulp_domain=domain) + try: + assert Repository.objects.using(SATELLITE_ALIAS).filter(pk=repo.pk).exists(), ( + "Repository created under a satellite-domain context should exist on the " + "satellite alias." + ) + assert not Repository.objects.using("default").filter(pk=repo.pk).exists(), ( + "Repository created under a satellite-domain context must NOT exist on " + "'default' -- routing to the wrong alias would silently duplicate/leak data." + ) + finally: + Repository.objects.using(SATELLITE_ALIAS).filter(pk=repo.pk).delete() + + def test_instance_hint_routes_without_contextvar(self): + with _satellite_domain(_suffix="instancehint") as domain: + with with_domain(domain): + repo = Repository.objects.create(name=f"{domain.name}-repo", pulp_domain=domain) + try: + repo_fresh = ( + Repository.objects.using(SATELLITE_ALIAS) + .select_related("pulp_domain") + .get(pk=repo.pk) + ) + repo_fresh.description = "updated via instance hint, no ContextVar" + repo_fresh.save() + assert ( + Repository.objects.using(SATELLITE_ALIAS).get(pk=repo.pk).description + == "updated via instance hint, no ContextVar" + ) + finally: + Repository.objects.using(SATELLITE_ALIAS).filter(pk=repo.pk).delete() + + def test_control_plane_model_always_routes_to_default(self): + with _satellite_domain(_suffix="controlplane") as domain: + with with_domain(domain): + task = Task.objects.create(name="test-task", state=TASK_STATES.WAITING) + try: + assert Task.objects.using("default").filter(pk=task.pk).exists() + assert not Task.objects.using(SATELLITE_ALIAS).filter(pk=task.pk).exists() + finally: + Task.objects.using("default").filter(pk=task.pk).delete() + + +class TestRouterInstanceHintSafety: + def test_remote_artifact_construction_does_not_recurse(self): + with _satellite_domain(_suffix="norecursion") as domain: + with with_domain(domain): + remote = Remote.objects.create(name="ki27-remote", url="https://example.com") + ca = ContentArtifact(relative_path="ki27/path") + try: + ra = RemoteArtifact(remote=remote, url=f"{remote.url}/x", content_artifact=ca) + except RecursionError: + pytest.fail( + "PulpDomainRouter._resolve_db recursed while constructing a " + "RemoteArtifact with a preceding unsaved FK" + ) + try: + assert ra.pulp_domain_id == domain.pk + finally: + Remote.objects.using(SATELLITE_ALIAS).filter(pk=remote.pk).delete() + + def test_relation_access_does_not_issue_extra_domain_query(self, django_assert_num_queries): + from pulp_file.app.models import FileRemote, FileRepository + + remote = FileRemote.objects.create(name="ki27-cast-remote") + repository = FileRepository.objects.create(name="ki27-cast-repo", remote=remote) + try: + with django_assert_num_queries(1): + fetched = Repository.objects.get(pk=repository.pk) + with django_assert_num_queries(1): + fetched = fetched.cast() + with django_assert_num_queries(1): + assert fetched.remote.pk == remote.pk + finally: + repository.delete() + remote.delete() + + +class TestGracefulDegradation: + def test_503_when_satellite_unreachable(self): + from pulpcore.middleware import DomainMiddleware + + with _satellite_domain(_suffix="unreachable") as domain: + request = mock.Mock(method="GET") + with mock.patch("pulpcore.middleware.connections") as mock_connections: + mock_connections.__getitem__.return_value.ensure_connection.side_effect = ( + OperationalError("could not connect") + ) + response = DomainMiddleware._degraded_response(request, domain) + + assert response is not None + assert response.status_code == 503 + assert domain.name in response.content.decode() + + def test_no_503_when_satellite_reachable(self): + with _satellite_domain(_suffix="reachable") as domain: + from pulpcore.middleware import DomainMiddleware + + request = mock.Mock(method="GET") + response = DomainMiddleware._degraded_response(request, domain) + assert response is None + + def test_503_rejects_writes_to_moving_domain(self): + with _satellite_domain(_suffix="moving", moving=True) as domain: + from pulpcore.middleware import DomainMiddleware + + write_request = mock.Mock(method="POST") + response = DomainMiddleware._degraded_response(write_request, domain) + assert response is not None + assert response.status_code == 503 + + read_request = mock.Mock(method="GET") + assert DomainMiddleware._degraded_response(read_request, domain) is None + + def test_task_dispatch_skips_moving_domain(self): + from pulpcore.tasking.worker import PulpcoreWorker + + with _satellite_domain(_suffix="taskmoving", moving=True) as domain: + with with_domain(domain): + task = Task.objects.create(name="test-task", state=TASK_STATES.WAITING) + try: + worker = mock.Mock(spec=PulpcoreWorker) + assert PulpcoreWorker.is_domain_available(worker, task) is False + finally: + Task.objects.using("default").filter(pk=task.pk).delete() + + def test_task_dispatch_skips_unreachable_satellite(self): + from pulpcore.tasking.worker import PulpcoreWorker + + with _satellite_domain(_suffix="taskunreachable") as domain: + with with_domain(domain): + task = Task.objects.create(name="test-task", state=TASK_STATES.WAITING) + try: + worker = mock.Mock(spec=PulpcoreWorker) + with mock.patch("pulpcore.tasking.worker.connections") as mock_connections: + mock_connections.__getitem__.return_value.ensure_connection.side_effect = ( + OperationalError("could not connect") + ) + assert PulpcoreWorker.is_domain_available(worker, task) is False + finally: + Task.objects.using("default").filter(pk=task.pk).delete() + + def test_task_dispatch_allows_healthy_domain(self): + from pulpcore.tasking.worker import PulpcoreWorker + + with _satellite_domain(_suffix="taskhealthy") as domain: + with with_domain(domain): + task = Task.objects.create(name="test-task", state=TASK_STATES.WAITING) + try: + worker = mock.Mock(spec=PulpcoreWorker) + assert PulpcoreWorker.is_domain_available(worker, task) is True + finally: + Task.objects.using("default").filter(pk=task.pk).delete() diff --git a/pulpcore/tests/unit/test_reconciliation.py b/pulpcore/tests/unit/test_reconciliation.py new file mode 100644 index 00000000000..8cabccb40ef --- /dev/null +++ b/pulpcore/tests/unit/test_reconciliation.py @@ -0,0 +1,135 @@ +from datetime import timedelta + +import pytest +from django.core.management import call_command +from django.utils import timezone + +from pulpcore.app.contexts import with_domain, with_task_context +from pulpcore.app.models import CreatedResource, Domain, Task +from pulpcore.app.tasks.reconciliation import reconcile_cross_plane_references + +from pulp_file.app.models import FileRepository + +from .test_multi_database_routing import SATELLITE_ALIAS, requires_multi_db + +pytestmark = [requires_multi_db, pytest.mark.django_db(databases=["default", SATELLITE_ALIAS])] + + +@pytest.fixture +def satellite_domain(): + domain = Domain.objects.create( + name="reconcile-test-domain", + storage_class="pulpcore.app.models.storage.FileSystem", + storage_settings={"location": "/tmp/reconcile-test-domain"}, + database_alias=SATELLITE_ALIAS, + ) + yield domain + domain.delete() + + +@pytest.fixture +def task(): + t = Task.objects.create(name="reconcile-test-task") + yield t + t.delete() + + +def _backdate(created_resource, minutes): + CreatedResource.objects.using("default").filter(pk=created_resource.pk).update( + pulp_last_updated=timezone.now() - timedelta(minutes=minutes) + ) + created_resource.refresh_from_db() + + +class TestReconcileCrossPlaneReferences: + def test_healthy_cross_plane_reference_is_not_flagged(self, satellite_domain, task): + with with_task_context(task), with_domain(satellite_domain): + repo = FileRepository.objects.create( + name="reconcile-healthy-repo", pulp_domain=satellite_domain + ) + cr = CreatedResource.objects.create(content_object=repo) + _backdate(cr, minutes=120) + + report = reconcile_cross_plane_references(grace_period_minutes=60) + + assert report["checked"] >= 1 + assert report["orphaned"] == 0 + assert str(cr.pk) not in {o["pk"] for o in report["orphans"]} + + def test_orphaned_reference_is_detected_but_not_purged_by_default(self, satellite_domain, task): + with with_task_context(task), with_domain(satellite_domain): + repo = FileRepository.objects.create( + name="reconcile-orphan-repo", pulp_domain=satellite_domain + ) + cr = CreatedResource.objects.create(content_object=repo) + repo.delete(using=SATELLITE_ALIAS) + _backdate(cr, minutes=120) + + report = reconcile_cross_plane_references(grace_period_minutes=60, purge_after_days=0) + + assert report["orphaned"] == 1 + assert report["purged"] == 0 + assert CreatedResource.objects.using("default").filter(pk=cr.pk).exists() + orphan = report["orphans"][0] + assert orphan["pk"] == str(cr.pk) + assert orphan["alias"] == SATELLITE_ALIAS + + def test_orphaned_reference_within_grace_period_is_skipped(self, satellite_domain, task): + with with_task_context(task), with_domain(satellite_domain): + repo = FileRepository.objects.create( + name="reconcile-fresh-orphan-repo", pulp_domain=satellite_domain + ) + cr = CreatedResource.objects.create(content_object=repo) + repo.delete(using=SATELLITE_ALIAS) + + report = reconcile_cross_plane_references(grace_period_minutes=60) + + assert str(cr.pk) not in {o["pk"] for o in report["orphans"]} + + def test_purge_after_days_deletes_old_confirmed_orphans(self, satellite_domain, task): + with with_task_context(task), with_domain(satellite_domain): + repo = FileRepository.objects.create( + name="reconcile-purge-repo", pulp_domain=satellite_domain + ) + cr = CreatedResource.objects.create(content_object=repo) + repo.delete(using=SATELLITE_ALIAS) + _backdate(cr, minutes=60 * 24 * 10) + + report = reconcile_cross_plane_references( + grace_period_minutes=60, purge_after_days=7, dry_run=False + ) + + assert report["orphaned"] == 1 + assert report["purged"] == 1 + assert not CreatedResource.objects.using("default").filter(pk=cr.pk).exists() + + def test_dry_run_never_purges(self, satellite_domain, task): + with with_task_context(task), with_domain(satellite_domain): + repo = FileRepository.objects.create( + name="reconcile-dry-run-repo", pulp_domain=satellite_domain + ) + cr = CreatedResource.objects.create(content_object=repo) + repo.delete(using=SATELLITE_ALIAS) + _backdate(cr, minutes=60 * 24 * 10) + + report = reconcile_cross_plane_references( + grace_period_minutes=60, purge_after_days=7, dry_run=True + ) + + assert report["orphaned"] == 1 + assert report["purged"] == 0 + assert CreatedResource.objects.using("default").filter(pk=cr.pk).exists() + + def test_management_command_reports_orphans(self, satellite_domain, task, capsys): + with with_task_context(task), with_domain(satellite_domain): + repo = FileRepository.objects.create( + name="reconcile-cmd-repo", pulp_domain=satellite_domain + ) + cr = CreatedResource.objects.create(content_object=repo) + repo.delete(using=SATELLITE_ALIAS) + _backdate(cr, minutes=120) + + call_command("reconcile-cross-plane-references", "--grace-period-minutes", "60") + + out = capsys.readouterr().out + assert "1 orphan(s)" in out or "found 1 orphan" in out.lower()