diff --git a/config/settings/base.py b/config/settings/base.py index 970acd14..469e2953 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -694,3 +694,6 @@ # ------------------------------------------------------------------------------ MAX_PAGES = env.int("MAX_PAGES", default=50) GRAFT_DEBUG = env.bool("GRAFT_DEBUG", default=False) + +# STATS API +UPLOAD_WINDOW_DAYS = env.int("UPLOAD_WINDOW_DAYS", default=90) diff --git a/config/urls.py b/config/urls.py index 0727eded..656159c6 100644 --- a/config/urls.py +++ b/config/urls.py @@ -43,6 +43,7 @@ from documentcloud.drf_bulk.routers import BulkDefaultRouter, BulkRouterMixin from documentcloud.entities.views import EntityOccurrenceViewSet, EntityViewSet from documentcloud.flatpages.views import FlatPageViewSet +from documentcloud.organizations.stats_api.views import OrganizationStatsViewSet from documentcloud.organizations.views import OrganizationViewSet from documentcloud.projects.views import ( CollaborationViewSet, @@ -50,6 +51,7 @@ ProjectViewSet, ) from documentcloud.statistics.views import StatisticsViewSet +from documentcloud.users.stats_api.views import UserStatsViewSet from documentcloud.users.views import MessageView, UserViewSet @@ -94,6 +96,10 @@ class BulkNestedDefaultRouter(BulkRouterMixin, NestedDefaultRouter): router.register("documents/search/saved", SavedSearchViewSet, basename="saved_search") +stats_router = BulkDefaultRouter() +stats_router.register("users", UserStatsViewSet, basename="user-stats") +stats_router.register("organizations", OrganizationStatsViewSet, basename="org-stats") + urlpatterns = [ path("", RedirectView.as_view(url="/api/"), name="index"), path(settings.ADMIN_URL, admin.site.urls), @@ -138,6 +144,7 @@ class BulkNestedDefaultRouter(BulkRouterMixin, NestedDefaultRouter): path( "addons/dashboard/scraper/", scraper_dashboard, name="addon-scraper-dashboard" ), + path("stats_api/", include(stats_router.urls)), ] if "debug_toolbar" in settings.INSTALLED_APPS: diff --git a/documentcloud/organizations/migrations/0023_organizationstats_and_more.py b/documentcloud/organizations/migrations/0023_organizationstats_and_more.py new file mode 100644 index 00000000..dff626ca --- /dev/null +++ b/documentcloud/organizations/migrations/0023_organizationstats_and_more.py @@ -0,0 +1,264 @@ +# Generated by Django 5.2.15 on 2026-08-03 15:06 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("organizations", "0022_organization_created_at_organization_updated_at"), + ] + + operations = [ + migrations.CreateModel( + name="OrganizationStats", + fields=[ + ( + "organization", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="stats", + serialize=False, + to=settings.SQUARELET_ORGANIZATION_MODEL, + ), + ), + ( + "last_upload_at", + models.DateTimeField(blank=True, db_index=True, null=True), + ), + ], + ), + migrations.AlterField( + model_name="organization", + name="document_language", + field=models.CharField( + blank=True, + choices=[ + ("afr", "Afrikaans"), + ("amh", "Amharic"), + ("ara", "Arabic"), + ("asm", "Assamese"), + ("aze", "Azerbaijani"), + ("aze_cyrl", "Azerbaijani - Cyrillic"), + ("bel", "Belarusian"), + ("ben", "Bengali"), + ("bod", "Tibetan"), + ("bos", "Bosnian"), + ("bul", "Bulgarian"), + ("cat", "Catalan; Valencian"), + ("ceb", "Cebuano"), + ("ces", "Czech"), + ("zho", "chi_sim"), + ("tra", "chi_tra"), + ("chr", "Cherokee"), + ("cym", "Welsh"), + ("dan", "Danish"), + ("deu", "German"), + ("dzo", "Dzongkha"), + ("ell", "Greek"), + ("eng", "English"), + ("enm", "Middle English"), + ("epo", "Esperanto"), + ("est", "Estonian"), + ("eus", "Basque"), + ("fas", "Persian"), + ("fin", "Finnish"), + ("fra", "French"), + ("frk", "German Fraktur"), + ("frm", "Middle French"), + ("gle", "Irish"), + ("glg", "Galician"), + ("grc", "Ancient Greek"), + ("guj", "Gujarati"), + ("hat", "Haitian; Haitian Creole"), + ("heb", "Hebrew"), + ("hin", "Hindi"), + ("hrv", "Croatian"), + ("hun", "Hungarian"), + ("iku", "Inuktitut"), + ("ind", "Indonesian"), + ("isl", "Icelandic"), + ("ita", "Italian"), + ("ita_old", "Italian - Old"), + ("jav", "Javanese"), + ("jpn", "Japanese"), + ("kan", "Kannada"), + ("kat", "Georgian"), + ("kat_old", "Georgian - Old"), + ("kaz", "Kazakh"), + ("khm", "Central Khmer"), + ("kir", "Kirghiz; Kyrgyz"), + ("kor", "Korean"), + ("kur", "Kurdish"), + ("lao", "Lao"), + ("lat", "Latin"), + ("lav", "Latvian"), + ("lit", "Lithuanian"), + ("mal", "Malayalam"), + ("mar", "Marathi"), + ("mkd", "Macedonian"), + ("mlt", "Maltese"), + ("msa", "Malay"), + ("mya", "Burmese"), + ("nep", "Nepali"), + ("nld", "Dutch; Flemish"), + ("nor", "Norwegian"), + ("ori", "Oriya"), + ("pan", "Panjabi; Punjabi"), + ("pol", "Polish"), + ("por", "Portuguese"), + ("pus", "Pushto; Pashto"), + ("ron", "Romanian; Moldavian; Moldovan"), + ("rus", "Russian"), + ("san", "Sanskrit"), + ("sin", "Sinhala; Sinhalese"), + ("slk", "Slovak"), + ("slv", "Slovenian"), + ("spa", "Spanish; Castilian"), + ("spa_old", "Spanish; Castilian - Old"), + ("sqi", "Albanian"), + ("srp", "Serbian"), + ("srp_latn", "Serbian - Latin"), + ("swa", "Swahili"), + ("swe", "Swedish"), + ("syr", "Syriac"), + ("tam", "Tamil"), + ("tel", "Telugu"), + ("tgk", "Tajik"), + ("tgl", "Tagalog"), + ("tha", "Thai"), + ("tir", "Tigrinya"), + ("tur", "Turkish"), + ("uig", "Uighur; Uyghur"), + ("ukr", "Ukrainian"), + ("urd", "Urdu"), + ("uzb", "Uzbek"), + ("uzb_cyrl", "Uzbek - Cyrillic"), + ("vie", "Vietnamese"), + ("yid", "Yiddish"), + ], + default="eng", + help_text="The default document language for user's in this organization", + max_length=8, + verbose_name="document language", + ), + ), + migrations.AlterField( + model_name="organization", + name="language", + field=models.CharField( + blank=True, + choices=[ + ("afr", "Afrikaans"), + ("amh", "Amharic"), + ("ara", "Arabic"), + ("asm", "Assamese"), + ("aze", "Azerbaijani"), + ("aze_cyrl", "Azerbaijani - Cyrillic"), + ("bel", "Belarusian"), + ("ben", "Bengali"), + ("bod", "Tibetan"), + ("bos", "Bosnian"), + ("bul", "Bulgarian"), + ("cat", "Catalan; Valencian"), + ("ceb", "Cebuano"), + ("ces", "Czech"), + ("zho", "chi_sim"), + ("tra", "chi_tra"), + ("chr", "Cherokee"), + ("cym", "Welsh"), + ("dan", "Danish"), + ("deu", "German"), + ("dzo", "Dzongkha"), + ("ell", "Greek"), + ("eng", "English"), + ("enm", "Middle English"), + ("epo", "Esperanto"), + ("est", "Estonian"), + ("eus", "Basque"), + ("fas", "Persian"), + ("fin", "Finnish"), + ("fra", "French"), + ("frk", "German Fraktur"), + ("frm", "Middle French"), + ("gle", "Irish"), + ("glg", "Galician"), + ("grc", "Ancient Greek"), + ("guj", "Gujarati"), + ("hat", "Haitian; Haitian Creole"), + ("heb", "Hebrew"), + ("hin", "Hindi"), + ("hrv", "Croatian"), + ("hun", "Hungarian"), + ("iku", "Inuktitut"), + ("ind", "Indonesian"), + ("isl", "Icelandic"), + ("ita", "Italian"), + ("ita_old", "Italian - Old"), + ("jav", "Javanese"), + ("jpn", "Japanese"), + ("kan", "Kannada"), + ("kat", "Georgian"), + ("kat_old", "Georgian - Old"), + ("kaz", "Kazakh"), + ("khm", "Central Khmer"), + ("kir", "Kirghiz; Kyrgyz"), + ("kor", "Korean"), + ("kur", "Kurdish"), + ("lao", "Lao"), + ("lat", "Latin"), + ("lav", "Latvian"), + ("lit", "Lithuanian"), + ("mal", "Malayalam"), + ("mar", "Marathi"), + ("mkd", "Macedonian"), + ("mlt", "Maltese"), + ("msa", "Malay"), + ("mya", "Burmese"), + ("nep", "Nepali"), + ("nld", "Dutch; Flemish"), + ("nor", "Norwegian"), + ("ori", "Oriya"), + ("pan", "Panjabi; Punjabi"), + ("pol", "Polish"), + ("por", "Portuguese"), + ("pus", "Pushto; Pashto"), + ("ron", "Romanian; Moldavian; Moldovan"), + ("rus", "Russian"), + ("san", "Sanskrit"), + ("sin", "Sinhala; Sinhalese"), + ("slk", "Slovak"), + ("slv", "Slovenian"), + ("spa", "Spanish; Castilian"), + ("spa_old", "Spanish; Castilian - Old"), + ("sqi", "Albanian"), + ("srp", "Serbian"), + ("srp_latn", "Serbian - Latin"), + ("swa", "Swahili"), + ("swe", "Swedish"), + ("syr", "Syriac"), + ("tam", "Tamil"), + ("tel", "Telugu"), + ("tgk", "Tajik"), + ("tgl", "Tagalog"), + ("tha", "Thai"), + ("tir", "Tigrinya"), + ("tur", "Turkish"), + ("uig", "Uighur; Uyghur"), + ("ukr", "Ukrainian"), + ("urd", "Urdu"), + ("uzb", "Uzbek"), + ("uzb_cyrl", "Uzbek - Cyrillic"), + ("vie", "Vietnamese"), + ("yid", "Yiddish"), + ], + default="eng", + help_text="The default interface language for user's in this organization", + max_length=8, + verbose_name="language", + ), + ), + ] diff --git a/documentcloud/organizations/models.py b/documentcloud/organizations/models.py index 422a7663..24470a5e 100644 --- a/documentcloud/organizations/models.py +++ b/documentcloud/organizations/models.py @@ -19,6 +19,9 @@ from documentcloud.core.fields import AutoCreatedField, AutoLastModifiedField from documentcloud.organizations.exceptions import InsufficientAICreditsError from documentcloud.organizations.querysets import OrganizationQuerySet +from documentcloud.organizations.stats_api.models import ( # pylint: disable=unused-import + OrganizationStats, +) logger = logging.getLogger(__name__) diff --git a/documentcloud/organizations/stats_api/__init__.py b/documentcloud/organizations/stats_api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/documentcloud/organizations/stats_api/models.py b/documentcloud/organizations/stats_api/models.py new file mode 100644 index 00000000..df5510c6 --- /dev/null +++ b/documentcloud/organizations/stats_api/models.py @@ -0,0 +1,18 @@ +# Django +from django.db import models + + +class OrganizationStats(models.Model): + organization = models.OneToOneField( + "organizations.Organization", + on_delete=models.CASCADE, + related_name="stats", + primary_key=True, + ) + last_upload_at = models.DateTimeField(null=True, blank=True, db_index=True) + + class Meta: + app_label = "organizations" + + def __str__(self): + return f"Stats for organization {self.organization_id}" diff --git a/documentcloud/organizations/stats_api/serializers.py b/documentcloud/organizations/stats_api/serializers.py new file mode 100644 index 00000000..c5372002 --- /dev/null +++ b/documentcloud/organizations/stats_api/serializers.py @@ -0,0 +1,35 @@ +# Django +from django.utils import timezone +from rest_framework import serializers + +# DocumentCloud +from documentcloud.organizations.stats_api.models import OrganizationStats + + +class OrganizationStatsSerializer(serializers.ModelSerializer): + uuid = serializers.UUIDField(source="organization.uuid", read_only=True) + total_documents = serializers.IntegerField(read_only=True) + days_since_last_upload = serializers.SerializerMethodField() + recent_upload_count = serializers.SerializerMethodField( + help_text="Documents uploaded by the org within the configured window " + "(UPLOAD_WINDOW_DAYS, defaults to 90)." + ) + + class Meta: + model = OrganizationStats + fields = [ + "uuid", + "total_documents", + "last_upload_at", + "days_since_last_upload", + "recent_upload_count", + ] + read_only_fields = fields + + def get_days_since_last_upload(self, obj): + if obj.last_upload_at is None: + return None + return (timezone.now() - obj.last_upload_at).days + + def get_recent_upload_count(self, obj): + return getattr(obj, "recent_upload_count", None) diff --git a/documentcloud/organizations/stats_api/views.py b/documentcloud/organizations/stats_api/views.py new file mode 100644 index 00000000..4bab06a5 --- /dev/null +++ b/documentcloud/organizations/stats_api/views.py @@ -0,0 +1,103 @@ +# Django +from django.conf import settings +from django.db.models import Count, Q +from django.utils import timezone +from django.utils.dateparse import parse_datetime +from rest_framework import mixins, viewsets +from rest_framework.decorators import action +from rest_framework.permissions import IsAdminUser +from rest_framework.response import Response + +# Standard Library +from datetime import timedelta + +# Third Party +from django_filters import rest_framework as django_filters + +# DocumentCloud +from documentcloud.core.pagination import CursorCountPagination +from documentcloud.documents.choices import Status +from documentcloud.organizations.stats_api.models import OrganizationStats +from documentcloud.organizations.stats_api.serializers import ( + OrganizationStatsSerializer, +) + + +class OrganizationStatsViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + serializer_class = OrganizationStatsSerializer + permission_classes = [IsAdminUser] + filter_backends = [django_filters.DjangoFilterBackend] + pagination_class = CursorCountPagination + lookup_field = "organization__uuid" + lookup_url_kwarg = "uuid" + + class Filter(django_filters.FilterSet): + uploaded_within_days = django_filters.NumberFilter( + method="filter_uploaded_within_days", + label="Uploaded in last N days", + help_text=( + "Return orgs whose most recent upload was within the last N days." + ), + ) + + def filter_uploaded_within_days(self, queryset, _name, value): + days = int(value) + if days < 0: + return queryset.none() + cutoff = timezone.now() - timedelta(days=days) + return queryset.filter(last_upload_at__gte=cutoff) + + class Meta: + model = OrganizationStats + fields = [] + + filterset_class = Filter + + def get_queryset(self): + cutoff = timezone.now() - timedelta(days=settings.UPLOAD_WINDOW_DAYS) + return ( + OrganizationStats.objects.select_related("organization") + .filter(organization__individual=False) + .annotate( + total_documents=Count( + "organization__documents", + filter=~Q(organization__documents__status=Status.deleted), + distinct=True, + ), + recent_upload_count=Count( + "organization__documents", + filter=Q(organization__documents__created_at__gte=cutoff) + & ~Q(organization__documents__status=Status.deleted), + distinct=True, + ), + ) + ) + + @action(detail=False, methods=["get"]) + def aged_out(self, request): + """Orgs with a document that crossed the window boundary since `since`, + so their upload count has dropped and needs re-syncing.""" + since = request.query_params.get("since") + if not since: + return Response({"error": "since query param is required"}, status=400) + since_dt = parse_datetime(since) + if since_dt is None: + return Response({"error": "since must be an ISO 8601 datetime"}, status=400) + + win = timedelta(days=settings.UPLOAD_WINDOW_DAYS) + now = timezone.now() + qs = ( + self.get_queryset() + .filter( + organization__documents__created_at__gte=since_dt - win, + organization__documents__created_at__lt=now - win, + ) + .distinct() + ) + + page = self.paginate_queryset(qs) + return Response(self.get_serializer(page, many=True).data) diff --git a/documentcloud/organizations/tests/test_models.py b/documentcloud/organizations/tests/test_models.py index 9916230c..ae84d5f1 100644 --- a/documentcloud/organizations/tests/test_models.py +++ b/documentcloud/organizations/tests/test_models.py @@ -71,7 +71,7 @@ def test_merge_fks(self): if f.is_relation and f.auto_created ] ) - == 8 + == 9 ) # Many to many relations defined on the Organization model assert ( diff --git a/documentcloud/users/migrations/0013_userstats_alter_user_document_language_and_more.py b/documentcloud/users/migrations/0013_userstats_alter_user_document_language_and_more.py new file mode 100644 index 00000000..e2ac55ec --- /dev/null +++ b/documentcloud/users/migrations/0013_userstats_alter_user_document_language_and_more.py @@ -0,0 +1,264 @@ +# Generated by Django 5.2.15 on 2026-08-01 21:22 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0012_default_pinned_projects"), + ] + + operations = [ + migrations.CreateModel( + name="UserStats", + fields=[ + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="stats", + serialize=False, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "last_upload_at", + models.DateTimeField(blank=True, db_index=True, null=True), + ), + ], + ), + migrations.AlterField( + model_name="user", + name="document_language", + field=models.CharField( + blank=True, + choices=[ + ("afr", "Afrikaans"), + ("amh", "Amharic"), + ("ara", "Arabic"), + ("asm", "Assamese"), + ("aze", "Azerbaijani"), + ("aze_cyrl", "Azerbaijani - Cyrillic"), + ("bel", "Belarusian"), + ("ben", "Bengali"), + ("bod", "Tibetan"), + ("bos", "Bosnian"), + ("bul", "Bulgarian"), + ("cat", "Catalan; Valencian"), + ("ceb", "Cebuano"), + ("ces", "Czech"), + ("zho", "chi_sim"), + ("tra", "chi_tra"), + ("chr", "Cherokee"), + ("cym", "Welsh"), + ("dan", "Danish"), + ("deu", "German"), + ("dzo", "Dzongkha"), + ("ell", "Greek"), + ("eng", "English"), + ("enm", "Middle English"), + ("epo", "Esperanto"), + ("est", "Estonian"), + ("eus", "Basque"), + ("fas", "Persian"), + ("fin", "Finnish"), + ("fra", "French"), + ("frk", "German Fraktur"), + ("frm", "Middle French"), + ("gle", "Irish"), + ("glg", "Galician"), + ("grc", "Ancient Greek"), + ("guj", "Gujarati"), + ("hat", "Haitian; Haitian Creole"), + ("heb", "Hebrew"), + ("hin", "Hindi"), + ("hrv", "Croatian"), + ("hun", "Hungarian"), + ("iku", "Inuktitut"), + ("ind", "Indonesian"), + ("isl", "Icelandic"), + ("ita", "Italian"), + ("ita_old", "Italian - Old"), + ("jav", "Javanese"), + ("jpn", "Japanese"), + ("kan", "Kannada"), + ("kat", "Georgian"), + ("kat_old", "Georgian - Old"), + ("kaz", "Kazakh"), + ("khm", "Central Khmer"), + ("kir", "Kirghiz; Kyrgyz"), + ("kor", "Korean"), + ("kur", "Kurdish"), + ("lao", "Lao"), + ("lat", "Latin"), + ("lav", "Latvian"), + ("lit", "Lithuanian"), + ("mal", "Malayalam"), + ("mar", "Marathi"), + ("mkd", "Macedonian"), + ("mlt", "Maltese"), + ("msa", "Malay"), + ("mya", "Burmese"), + ("nep", "Nepali"), + ("nld", "Dutch; Flemish"), + ("nor", "Norwegian"), + ("ori", "Oriya"), + ("pan", "Panjabi; Punjabi"), + ("pol", "Polish"), + ("por", "Portuguese"), + ("pus", "Pushto; Pashto"), + ("ron", "Romanian; Moldavian; Moldovan"), + ("rus", "Russian"), + ("san", "Sanskrit"), + ("sin", "Sinhala; Sinhalese"), + ("slk", "Slovak"), + ("slv", "Slovenian"), + ("spa", "Spanish; Castilian"), + ("spa_old", "Spanish; Castilian - Old"), + ("sqi", "Albanian"), + ("srp", "Serbian"), + ("srp_latn", "Serbian - Latin"), + ("swa", "Swahili"), + ("swe", "Swedish"), + ("syr", "Syriac"), + ("tam", "Tamil"), + ("tel", "Telugu"), + ("tgk", "Tajik"), + ("tgl", "Tagalog"), + ("tha", "Thai"), + ("tir", "Tigrinya"), + ("tur", "Turkish"), + ("uig", "Uighur; Uyghur"), + ("ukr", "Ukrainian"), + ("urd", "Urdu"), + ("uzb", "Uzbek"), + ("uzb_cyrl", "Uzbek - Cyrillic"), + ("vie", "Vietnamese"), + ("yid", "Yiddish"), + ], + default="eng", + help_text="The default language for documents uploaded by this user", + max_length=8, + verbose_name="document language", + ), + ), + migrations.AlterField( + model_name="user", + name="language", + field=models.CharField( + blank=True, + choices=[ + ("afr", "Afrikaans"), + ("amh", "Amharic"), + ("ara", "Arabic"), + ("asm", "Assamese"), + ("aze", "Azerbaijani"), + ("aze_cyrl", "Azerbaijani - Cyrillic"), + ("bel", "Belarusian"), + ("ben", "Bengali"), + ("bod", "Tibetan"), + ("bos", "Bosnian"), + ("bul", "Bulgarian"), + ("cat", "Catalan; Valencian"), + ("ceb", "Cebuano"), + ("ces", "Czech"), + ("zho", "chi_sim"), + ("tra", "chi_tra"), + ("chr", "Cherokee"), + ("cym", "Welsh"), + ("dan", "Danish"), + ("deu", "German"), + ("dzo", "Dzongkha"), + ("ell", "Greek"), + ("eng", "English"), + ("enm", "Middle English"), + ("epo", "Esperanto"), + ("est", "Estonian"), + ("eus", "Basque"), + ("fas", "Persian"), + ("fin", "Finnish"), + ("fra", "French"), + ("frk", "German Fraktur"), + ("frm", "Middle French"), + ("gle", "Irish"), + ("glg", "Galician"), + ("grc", "Ancient Greek"), + ("guj", "Gujarati"), + ("hat", "Haitian; Haitian Creole"), + ("heb", "Hebrew"), + ("hin", "Hindi"), + ("hrv", "Croatian"), + ("hun", "Hungarian"), + ("iku", "Inuktitut"), + ("ind", "Indonesian"), + ("isl", "Icelandic"), + ("ita", "Italian"), + ("ita_old", "Italian - Old"), + ("jav", "Javanese"), + ("jpn", "Japanese"), + ("kan", "Kannada"), + ("kat", "Georgian"), + ("kat_old", "Georgian - Old"), + ("kaz", "Kazakh"), + ("khm", "Central Khmer"), + ("kir", "Kirghiz; Kyrgyz"), + ("kor", "Korean"), + ("kur", "Kurdish"), + ("lao", "Lao"), + ("lat", "Latin"), + ("lav", "Latvian"), + ("lit", "Lithuanian"), + ("mal", "Malayalam"), + ("mar", "Marathi"), + ("mkd", "Macedonian"), + ("mlt", "Maltese"), + ("msa", "Malay"), + ("mya", "Burmese"), + ("nep", "Nepali"), + ("nld", "Dutch; Flemish"), + ("nor", "Norwegian"), + ("ori", "Oriya"), + ("pan", "Panjabi; Punjabi"), + ("pol", "Polish"), + ("por", "Portuguese"), + ("pus", "Pushto; Pashto"), + ("ron", "Romanian; Moldavian; Moldovan"), + ("rus", "Russian"), + ("san", "Sanskrit"), + ("sin", "Sinhala; Sinhalese"), + ("slk", "Slovak"), + ("slv", "Slovenian"), + ("spa", "Spanish; Castilian"), + ("spa_old", "Spanish; Castilian - Old"), + ("sqi", "Albanian"), + ("srp", "Serbian"), + ("srp_latn", "Serbian - Latin"), + ("swa", "Swahili"), + ("swe", "Swedish"), + ("syr", "Syriac"), + ("tam", "Tamil"), + ("tel", "Telugu"), + ("tgk", "Tajik"), + ("tgl", "Tagalog"), + ("tha", "Thai"), + ("tir", "Tigrinya"), + ("tur", "Turkish"), + ("uig", "Uighur; Uyghur"), + ("ukr", "Ukrainian"), + ("urd", "Urdu"), + ("uzb", "Uzbek"), + ("uzb_cyrl", "Uzbek - Cyrillic"), + ("vie", "Vietnamese"), + ("yid", "Yiddish"), + ], + default="eng", + help_text="The interface language for this user", + max_length=8, + verbose_name="language", + ), + ), + ] diff --git a/documentcloud/users/models.py b/documentcloud/users/models.py index f455f06e..1e0d6d51 100644 --- a/documentcloud/users/models.py +++ b/documentcloud/users/models.py @@ -11,6 +11,9 @@ # DocumentCloud from documentcloud.core.choices import Language from documentcloud.users.managers import UserManager +from documentcloud.users.stats_api.models import ( # pylint: disable=unused-import + UserStats, +) class User(SAUser): diff --git a/documentcloud/users/signals.py b/documentcloud/users/signals.py index 4a7cdd0f..0a8d2b61 100644 --- a/documentcloud/users/signals.py +++ b/documentcloud/users/signals.py @@ -1,6 +1,8 @@ # Django from django.contrib.auth.signals import user_logged_in +from django.db.models.signals import post_save from django.dispatch import receiver +from django.utils import timezone # DocumentCloud from documentcloud.addons.models import AddOn @@ -14,3 +16,27 @@ def default_addons(sender, user, request, **kwargs): if not user.active_addons.exists(): user.active_addons.set(AddOn.objects.filter(default=True)) + + +@receiver(post_save, dispatch_uid="documentcloud.users.signals.touch_last_upload") +def touch_last_upload(sender, instance, created, **kwargs): + """Record upload recency when a user creates a new document""" + # DocumentCloud + from documentcloud.documents.models import Document + from documentcloud.organizations.stats_api.models import OrganizationStats + from documentcloud.users.stats_api.models import UserStats + + if sender is not Document or not created: + return + + now = timezone.now() + UserStats.objects.update_or_create( + user_id=instance.user_id, + defaults={"last_upload_at": now}, + ) + + if not instance.organization.individual: + OrganizationStats.objects.update_or_create( + organization_id=instance.organization_id, + defaults={"last_upload_at": now}, + ) diff --git a/documentcloud/users/stats_api/__init__.py b/documentcloud/users/stats_api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/documentcloud/users/stats_api/models.py b/documentcloud/users/stats_api/models.py new file mode 100644 index 00000000..5f7cb50d --- /dev/null +++ b/documentcloud/users/stats_api/models.py @@ -0,0 +1,18 @@ +# Django +from django.db import models + + +class UserStats(models.Model): + user = models.OneToOneField( + "users.User", + on_delete=models.CASCADE, + related_name="stats", + primary_key=True, + ) + last_upload_at = models.DateTimeField(null=True, blank=True, db_index=True) + + class Meta: + app_label = "users" + + def __str__(self): + return f"Stats for user {self.user_id}" diff --git a/documentcloud/users/stats_api/serializers.py b/documentcloud/users/stats_api/serializers.py new file mode 100644 index 00000000..6cdc6726 --- /dev/null +++ b/documentcloud/users/stats_api/serializers.py @@ -0,0 +1,39 @@ +# Django +from django.utils import timezone +from rest_framework import serializers + +# DocumentCloud +from documentcloud.users.stats_api.models import UserStats + + +class UserStatsSerializer(serializers.ModelSerializer): + uuid = serializers.UUIDField(source="user.uuid", read_only=True) + total_documents = serializers.IntegerField(read_only=True) + days_since_last_upload = serializers.SerializerMethodField( + help_text="Number of days since the last time the user uploaded a document" + ) + last_login_at = serializers.DateTimeField(source="user.last_login", read_only=True) + recent_upload_count = serializers.SerializerMethodField( + help_text="Number of documents uploaded within the configured recent window " + "(UPLOAD_WINDOW_DAYS, currently defaults to 90)." + ) + + class Meta: + model = UserStats + fields = [ + "uuid", + "total_documents", + "last_upload_at", + "days_since_last_upload", + "last_login_at", + "recent_upload_count", + ] + read_only_fields = fields + + def get_days_since_last_upload(self, obj): + if obj.last_upload_at is None: + return None + return (timezone.now() - obj.last_upload_at).days + + def get_recent_upload_count(self, obj): + return getattr(obj, "recent_upload_count", None) diff --git a/documentcloud/users/stats_api/tests.py b/documentcloud/users/stats_api/tests.py new file mode 100644 index 00000000..e69de29b diff --git a/documentcloud/users/stats_api/views.py b/documentcloud/users/stats_api/views.py new file mode 100644 index 00000000..f58c6b3b --- /dev/null +++ b/documentcloud/users/stats_api/views.py @@ -0,0 +1,126 @@ +# Django +from django.conf import settings +from django.db.models import Count, Q +from django.utils import timezone +from django.utils.dateparse import parse_datetime +from rest_framework import mixins, viewsets +from rest_framework.decorators import action +from rest_framework.permissions import IsAdminUser +from rest_framework.response import Response + +# Standard Library +from datetime import timedelta + +# Third Party +from django_filters import rest_framework as django_filters + +# DocumentCloud +from documentcloud.core.pagination import CursorCountPagination +from documentcloud.documents.choices import Status +from documentcloud.users.stats_api.models import UserStats +from documentcloud.users.stats_api.serializers import UserStatsSerializer + + +class UserStatsViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + serializer_class = UserStatsSerializer + permission_classes = [IsAdminUser] + filter_backends = [django_filters.DjangoFilterBackend] + pagination_class = CursorCountPagination + lookup_field = "user__uuid" + lookup_url_kwarg = "uuid" + + class Filter(django_filters.FilterSet): + SINCE_FIELDS = ["last_upload_at", "user__last_login"] + + active_within_days = django_filters.NumberFilter( + method="filter_active_within_days", + label="Active in last N days (upload or login)", + help_text="Return users who uploaded or logged in within the last N days.", + ) + uploaded_within_days = django_filters.NumberFilter( + method="filter_uploaded_within_days", + label="Uploaded in last N days", + help_text="Return users who uploaded a document in the last N days.", + ) + logged_in_within_days = django_filters.NumberFilter( + method="filter_logged_in_within_days", + label="Logged in within last N days", + help_text=( + "Return users whose most recent login was within the last N days." + ), + ) + + def filter_active_within_days(self, queryset, _name, value): + days = int(value) + if days < 0: + return queryset.none() + cutoff = timezone.now() - timedelta(days=days) + query = Q() + for field in self.SINCE_FIELDS: + query |= Q(**{f"{field}__gt": cutoff}) + return queryset.filter(query) + + def filter_uploaded_within_days(self, queryset, _name, value): + days = int(value) + if days < 0: + return queryset.none() + cutoff = timezone.now() - timedelta(days=days) + return queryset.filter(last_upload_at__gte=cutoff) + + def filter_logged_in_within_days(self, queryset, _name, value): + days = int(value) + if days < 0: + return queryset.none() + cutoff = timezone.now() - timedelta(days=days) + return queryset.filter(user__last_login__gte=cutoff) + + class Meta: + model = UserStats + fields = [] + + filterset_class = Filter + + def get_queryset(self): + cutoff = timezone.now() - timedelta(days=settings.UPLOAD_WINDOW_DAYS) + return UserStats.objects.select_related("user").annotate( + total_documents=Count( + "user__documents", + filter=~Q(user__documents__status=Status.deleted), + distinct=True, + ), + recent_upload_count=Count( + "user__documents", + filter=Q(user__documents__created_at__gte=cutoff) + & ~Q(user__documents__status=Status.deleted), + distinct=True, + ), + ) + + @action(detail=False, methods=["get"]) + def aged_out(self, request): + """Users with a document that crossed the window boundary since `since`, + so their upload count has dropped and needs re-syncing.""" + since = request.query_params.get("since") + if not since: + return Response({"error": "since query param is required"}, status=400) + since_dt = parse_datetime(since) + if since_dt is None: + return Response({"error": "since must be an ISO 8601 datetime"}, status=400) + + win = timedelta(days=settings.UPLOAD_WINDOW_DAYS) + now = timezone.now() + qs = ( + self.get_queryset() + .filter( + user__documents__created_at__gte=since_dt - win, + user__documents__created_at__lt=now - win, + ) + .distinct() + ) + + page = self.paginate_queryset(qs) + return Response(self.get_serializer(page, many=True).data)