Merge Feature/pbs 26 20 into develop

admin/base/apps.py

@@ -4,3 +4,43 @@
 class BaseAdminAppConfig(BaseAppConfig):
     name = 'admin.base'
     label = 'admin_base'
+
+    def ready(self):
+        from django.contrib import admin
+        from django.core.exceptions import PermissionDenied
+        from dj_control_room_base.models import BasePanelPlaceholder
+        from django_celery_beat.admin import PeriodicTaskAdmin
+        from django_celery_beat.models import (
+            ClockedSchedule,
+            CrontabSchedule,
+            IntervalSchedule,
+            PeriodicTask,
+            SolarSchedule,
+        )
+
+        # dj_control_room_base is installed only for dj_celery_panel's templates/static; hide its demo panel
+        admin.site.unregister(BasePanelPlaceholder)
+
+        # Schedules are managed through CeleryConfig.beat_schedule, so only the periodic task list is exposed
+        admin.site.unregister((ClockedSchedule, CrontabSchedule, IntervalSchedule, SolarSchedule))
+
+        # Read-only list for Periodic Tasks admin panel: no actions, add, change or delete pages
+        class ReadOnlyPeriodicTaskAdmin(PeriodicTaskAdmin):
+            actions = None
+            list_display_links = None
+
+            def has_add_permission(self, request):
+                return False
+
+            def has_change_permission(self, request, obj=None):
+                return False
+
+            def has_delete_permission(self, request, obj=None):
+                return False
+
+            def change_view(self, request, object_id, form_url='', extra_context=None):
+                # the read-only view page still builds beat's form, which fails in admin
+                raise PermissionDenied
+
+        admin.site.unregister(PeriodicTask)
+        admin.site.register(PeriodicTask, ReadOnlyPeriodicTaskAdmin)

admin/base/celery_panel.py

@@ -0,0 +1,75 @@
+from dj_celery_panel.celery_utils import CeleryQueuesInspectBackend, QueueDetailPage, QueueListPage
+
+
+class RoutedTasksQueuesBackend(CeleryQueuesInspectBackend):
+    """dj-celery-panel queues backend that also lists the tasks routed to each queue
+    and the configured queues that no active worker consumes.
+
+    Routes are resolved the same way Celery does when sending a task, so explicit
+    queue= overrides passed to apply_async() at call time are not reflected.
+    """
+
+    BACKEND_DESCRIPTION = 'Queue bindings, routing configuration, routed tasks and unconsumed queues'
+
+    def get_queues(self):
+        page = super().get_queues()
+        # Workers report only the queues they consume, so a configured queue nobody consumes
+        # would be missing from the list while tasks routed to it pile up
+        listed = {queue['name'] for queue in page.queues}
+        unconsumed = [
+            self.describe_unconsumed_queue(queue)
+            for name, queue in sorted(self.app.amqp.queues.items())
+            if name not in listed
+        ]
+        return QueueListPage(queues=page.queues + unconsumed, error=page.error)
+
+    def get_queue_detail(self, queue_name):
+        page = super().get_queue_detail(queue_name)
+        queue, error = page.queue, page.error
+        if queue is None:
+            # membership check first: indexing Queues creates missing ones
+            if queue_name not in self.app.amqp.queues:
+                return page
+            queue = self.describe_unconsumed_queue(self.app.amqp.queues[queue_name])
+            error = 'No active worker consumes this queue'
+        routed_tasks = self.get_routed_tasks(queue_name, self.get_task_names(queue['workers']))
+        return QueueDetailPage(
+            queue={**queue, 'routed_tasks': routed_tasks},
+            error=error,
+        )
+
+    def describe_unconsumed_queue(self, queue):
+        broker_info = self._get_queue_length_from_broker(queue.name)
+        return {
+            'name': queue.name,
+            'exchange': queue.exchange.name,
+            'exchange_type': queue.exchange.type,
+            'routing_key': queue.routing_key,
+            'durable': queue.durable,
+            'auto_delete': queue.auto_delete,
+            'exclusive': queue.exclusive,
+            'arguments': queue.queue_arguments,
+            'workers': [],
+            'worker_details': [],
+            'message_count': broker_info.get('length'),
+            'consumer_count': broker_info.get('consumer_count'),
+            'broker_query_error': broker_info.get('error'),
+        }
+
+    def get_task_names(self, workers):
+        # The admin process registers only the task modules it happens to import,
+        # so ask the workers for the tasks they have registered: the ones consuming
+        # this queue, or all of them when nobody consumes it
+        registered = self.app.control.inspect(destination=workers or None).registered() or {}
+        names = {name for worker_tasks in registered.values() for name in worker_tasks}
+        return {name for name in names | set(self.app.tasks) if not name.startswith('celery.')}
+
+    def get_routed_tasks(self, queue_name, task_names):
+        router = self.app.amqp.router
+        routed_tasks = []
+        for name in task_names:
+            task = self.app.tasks.get(name)
+            options = dict(task._get_exec_options()) if task else {}
+            if router.route(options, name)['queue'].name == queue_name:
+                routed_tasks.append(name)
+        return sorted(routed_tasks)

admin/base/middleware.py

@@ -0,0 +1,24 @@
+from django.contrib.auth.views import redirect_to_login
+from django.core.exceptions import PermissionDenied
+from django.urls import reverse
+
+
+class AdminLoginRequiredMiddleware:
+    """Require an active staff user for every admin view not marked with ``login_not_required``.
+    Anonymous users are sent to the admin login page; signed-in users who aren't staff get a 403.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        return self.get_response(request)
+
+    def process_view(self, request, view_func, view_args, view_kwargs):
+        if not getattr(view_func, 'login_required', True):
+            return None
+        if not request.user.is_authenticated:
+            return redirect_to_login(request.get_full_path(), reverse('auth:login'))
+        if not request.user.is_staff:
+            raise PermissionDenied
+        return None

admin/base/settings/defaults.py

@@ -80,7 +80,10 @@
     'django.contrib.staticfiles',
 
     # 3rd party
+    'django_celery_beat',
     'django_celery_results',
+    'dj_control_room_base',
+    'dj_celery_panel',
     'webpack_loader',
     'guardian',
     'waffle',
@@ -108,6 +111,12 @@
 
 )
 
+DJ_CELERY_PANEL_SETTINGS = {
+    'queues_backend': 'admin.base.celery_panel.RoutedTasksQueuesBackend',
+    # Superusers always have access; users in these groups must also be staff
+    'ALLOWED_GROUPS': ['osf_admin', 'celery_panel'],
+}
+
 MIGRATION_MODULES = {
     'osf': None,
     'reviews': None,
@@ -148,6 +157,7 @@
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
+    'admin.base.middleware.AdminLoginRequiredMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'waffle.middleware.WaffleMiddleware',

admin/base/urls.py

@@ -12,6 +12,9 @@
         base_pattern,
         include([
             re_path(r'^$', views.home, name='home'),
+            re_path(r'^unprotected_views/$', views.unprotected_views, name='unprotected_views'),
+            re_path(r'^middleware_check/$', views.admin_auth_middleware_check, name='middleware_check'),
+            re_path(r'^admin/dj-celery-panel/', include('dj_celery_panel.urls')),
             re_path(r'^admin/', admin.site.urls),
             re_path(r'^provider_asset_files/', include('admin.provider_asset_files.urls', namespace='provider_asset_files')),
             re_path(r'^institution_asset_files/', include('admin.institution_asset_files.urls', namespace='institution_asset_files')),

admin/base/utils.py

@@ -4,7 +4,7 @@
 from osf.models import Subject, NodeLicense, Brand, CedarMetadataTemplate
 
 from django.core.exceptions import ValidationError, PermissionDenied
-from django.urls import reverse
+from django.urls import URLPattern, URLResolver, get_resolver, reverse
 from django.core.validators import RegexValidator, _lazy_re_compile
 from django.utils.http import urlencode
 from django.utils.translation import gettext_lazy as _
@@ -33,6 +33,38 @@ def osf_staff_check(user):
     return user and user.is_authenticated and user.is_staff
 
 
+def login_not_required(view_func):
+    """Let a view through AdminLoginRequiredMiddleware without a logged-in staff user.
+
+    For class-based views, decorate ``dispatch`` with ``method_decorator``.
+    """
+    view_func.login_required = False
+    return view_func
+
+
+def get_unprotected_url_patterns(url_patterns=None, route_prefix='/', name_prefix=''):
+    """List the URL patterns that AdminLoginRequiredMiddleware lets through without a logged-in staff user.
+
+    Each entry is a dict with the pattern's ``route``, namespaced URL ``name`` and the dotted path of its ``view``.
+    """
+    if url_patterns is None:
+        url_patterns = get_resolver().url_patterns
+    unprotected = []
+    for pattern in url_patterns:
+        route = route_prefix + str(pattern.pattern).lstrip('^').removesuffix('$')
+        if isinstance(pattern, URLResolver):
+            namespace = f'{name_prefix}{pattern.namespace}:' if pattern.namespace else name_prefix
+            unprotected += get_unprotected_url_patterns(pattern.url_patterns, route, namespace)
+        elif isinstance(pattern, URLPattern) and not getattr(pattern.callback, 'login_required', True):
+            view = getattr(pattern.callback, 'view_class', pattern.callback)
+            unprotected.append({
+                'route': route,
+                'name': f'{name_prefix}{pattern.name}' if pattern.name else '',
+                'view': f'{view.__module__}.{view.__qualname__}',
+            })
+    return unprotected
+
+
 def get_subject_rules(subjects_selected):
     """
     Take a list of subjects, and parse them into rules consistent with preprint provider

admin/base/views.py

@@ -1,10 +1,11 @@
 from django.core.exceptions import ObjectDoesNotExist
 from django.contrib.auth.decorators import user_passes_test
+from django.http import HttpResponse
 from django.shortcuts import render
 from django.views.generic import DetailView
 from django.views.defaults import page_not_found
 
-from admin.base.utils import osf_staff_check
+from admin.base.utils import get_unprotected_url_patterns, osf_staff_check
 
 
 @user_passes_test(osf_staff_check)
@@ -13,6 +14,18 @@ def home(request):
     return render(request, 'home.html', context)
 
 
+@user_passes_test(osf_staff_check)
+def unprotected_views(request):
+    context = {'unprotected_url_patterns': get_unprotected_url_patterns()}
+    return render(request, 'unprotected_views.html', context)
+
+
+def admin_auth_middleware_check(request):
+    # This view is used to check and see if the AdminLoginRequiredMiddleware is working correctly.
+    # It requires no permissions so that, if unauthenticated users can access it, the middleware is not working correctly.
+    return HttpResponse('AdminLoginRequiredMiddleware check: you are signed in as a staff user.')
+
+
 class GuidView(DetailView):
     def get(self, request, *args, **kwargs):
         try:

admin/common_auth/views.py

@@ -11,6 +11,7 @@
 
 from osf.models.user import OSFUser
 from osf.models import AdminProfile
+from admin.base.utils import login_not_required
 from admin.common_auth.forms import LoginForm, UserRegistrationForm, DeskUserForm, TwoFactorForm
 
 
@@ -19,6 +20,7 @@ class LoginView(FormView):
     redirect_field_name = REDIRECT_FIELD_NAME
     template_name = 'login.html'
 
+    @method_decorator(login_not_required)
     @method_decorator(csrf_protect)
     @method_decorator(never_cache)
     def dispatch(self, request, *args, **kwargs):
@@ -113,7 +115,6 @@ def get_success_url(self):
             redirect_to = reverse('home')
         return redirect_to
 
-
 def logout_user(request):
     logout(request)
     return redirect('auth:login')

admin/preprints/views.py

@@ -51,7 +51,7 @@
     REJECT_WITHDRAWAL,
     UNFLAG_SPAM,
 )
-from osf.utils.workflows import DefaultStates
+from osf.utils.workflows import DefaultStates, ReviewTriggers
 from osf.utils.permissions import API_CONTRIBUTOR_PERMISSIONS
 from website.search import search
 from website.files.utils import copy_files
@@ -844,8 +844,10 @@ def post(self, request, *args, **kwargs):
             messages.error(request, f'Preprint {preprint._id} is not withdrawn')
             return redirect(self.get_success_url())
 
-        withdraw_action = preprint.actions.filter(to_state='withdrawn').last()
-        last_action = preprint.actions.last()
+        # A spam report is logged with to_state='withdrawn' too, so state alone can't tell it from
+        # the withdrawal. Match the withdrawal by trigger
+        withdraw_action = preprint.actions.filter(trigger=ReviewTriggers.WITHDRAW.value).last()
+        last_action = preprint.actions.exclude(trigger=ReviewTriggers.REPORT_SPAM.value).last()
 
         preprint.withdrawal_justification = ''
         preprint.date_withdrawn = None

admin/share_reindex/views.py

@@ -5,6 +5,7 @@
 from osf.models import Guid
 from urllib.parse import urlencode
 from api.share.utils import get_not_indexed_guids_for_resource_with_no_indexed_guid, task__reindex_failed_or_not_indexed_resource_into_share
+from website.settings import CeleryConfig
 
 class FailedShareIndexedGuidList(PermissionRequiredMixin, ListView):
     paginate_by = 25
@@ -46,7 +47,7 @@ class FailedShareIndexedGuidReindex(PermissionRequiredMixin, View):
     def post(self, request, *args, **kwargs):
         resource_type = self.kwargs.get('resource_type')
         # reindex 100_000 guids in background task for specific resource_type and resource is public
-        task__reindex_failed_or_not_indexed_resource_into_share.delay(resource_type)
+        task__reindex_failed_or_not_indexed_resource_into_share.apply_async(args=(resource_type,), queue=CeleryConfig.task_low_queue)
         base_url = reverse('share_reindex:list')
         query_string = urlencode({'type': resource_type, 'status': 'indexing'})
         return redirect(f"{base_url}?{query_string}")

admin/templates/admin/dj_celery_panel/queue_detail.html

@@ -0,0 +1,32 @@
+{% extends "admin/dj_celery_panel/queue_detail.html" %}
+{% load i18n %}
+
+{% block panel_page_content %}
+{{ block.super }}
+{% if queue %}
+<dl class="dcr-dl dcr-mt-sm">
+  <div class="dcr-dl__row">
+    <dt class="dcr-dl__label">{% trans "Routed Tasks" %}</dt>
+    <dd class="dcr-dl__value">
+      {% if queue.routed_tasks %}
+      <strong>{{ queue.routed_tasks|length }}</strong> {% trans "tasks" %}
+      <details class="dcr-mt-sm">
+        <summary>{% trans "Show routed tasks" %}</summary>
+        <ul class="dcr-mt-sm">
+          {% for task in queue.routed_tasks %}
+          <li>
+            <a href="{% url 'dj_celery_panel:tasks' %}?search={{ task|urlencode }}" class="dcr-data-table__link">
+              <code class="dcr-code">{{ task }}</code>
+            </a>
+          </li>
+          {% endfor %}
+        </ul>
+      </details>
+      {% else %}
+      <span class="dcr-text-muted">{% trans "No tasks are routed to this queue" %}</span>
+      {% endif %}
+    </dd>
+  </div>
+</dl>
+{% endif %}
+{% endblock %}

admin/templates/base.html

@@ -348,6 +348,9 @@
           {% if perms.osf.change_maintenancestate %}
             <li><a href="{% url 'maintenance:display' %}"><i class='fa fa-link'></i> <span>Maintenance Alerts</span></a></li>
           {% endif %}
+          {% if user.is_staff %}
+            <li><a href="{% url 'unprotected_views' %}"><i class='fa fa-link'></i> <span>Unprotected Views</span></a></li>
+          {% endif %}
           </ul><!-- /.sidebar-menu -->
         </section>
         <!-- /.sidebar -->

admin/templates/notifications/notification_campaing_create.html

@@ -183,6 +183,7 @@ <h4>Recipient Filters</h4>
                         <input
                             type="checkbox"
                             id="exclude-non-contributors"
+                            checked
                         >
                         Exclude non-contributors
                     </label>

admin/templates/register.html

@@ -36,6 +36,7 @@ <h4><strong>Select Groups for Permissions</strong></h4>
                 <p><strong>metrics_only:</strong><br>User can only view metrics on the OSF Admin page</p>
                 <p><strong>read_only</strong><br>User can view details of nodes, preprints, users, meetings, metrics, and spam, but cannot make changes</p>
                 <p><strong>osf_admin</strong><br>User make changes to nodes, preprints, users, meetings, and mark spam</p>
+                <p><strong>celery_panel</strong><br>User can only view Celery monitoring pages in Django admin</p>
 
                 <p>{{ form.group_perms }}</p>
                 {% if form.group_perms.errors %}<span class="text-danger"> {{ form.group_perms.errors }}</span>{% endif %}