From a377a991d33f056a351e80e6e737ca54bc822539 Mon Sep 17 00:00:00 2001 From: hcphat Date: Tue, 28 Apr 2026 11:18:00 +0700 Subject: [PATCH 01/41] =?UTF-8?q?ref=202.2.=E3=82=A2=E3=83=89=E3=82=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AB?= =?UTF-8?q?=E7=B5=A1=E3=82=80=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E6=99=82=E3=81=AE=E3=82=AF=E3=82=AA=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=AE=A1=E7=90=86=20:=20Commit=20code=20and=20UT=20handle=20qu?= =?UTF-8?q?ota=20for=20move/copy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/base/views.py | 3 + addons/osfstorage/decorators.py | 3 +- .../tests/test_views_copy_move_quota.py | 865 ++++++++++++++++++ addons/osfstorage/views.py | 141 ++- tests/test_addons.py | 8 +- tests/test_quota_move_copy.py | 248 +++++ website/static/js/fangorn.js | 122 +++ .../en/LC_MESSAGES/js_messages.po | 12 + .../ja/LC_MESSAGES/js_messages.po | 12 + website/translations/js_messages.pot | 12 + website/util/quota.py | 113 +++ 11 files changed, 1532 insertions(+), 7 deletions(-) create mode 100644 addons/osfstorage/tests/test_views_copy_move_quota.py create mode 100644 tests/test_quota_move_copy.py diff --git a/addons/base/views.py b/addons/base/views.py index c1bb834b7b0..e6f75bbb5e7 100644 --- a/addons/base/views.py +++ b/addons/base/views.py @@ -433,6 +433,9 @@ def get_auth(auth, **kwargs): credentials = node.serialize_waterbutler_credentials(provider_name) waterbutler_settings = node.serialize_waterbutler_settings(provider_name) + if is_node_process and provider_settings: + waterbutler_settings['max_file_size'] = getattr(provider_settings.config, 'max_file_size', None) + if not is_node_process: # for only location_id value storage = ExportDataLocation.objects.get(pk=location_id) diff --git a/addons/osfstorage/decorators.py b/addons/osfstorage/decorators.py index decf4489ee5..b946976f5a8 100644 --- a/addons/osfstorage/decorators.py +++ b/addons/osfstorage/decorators.py @@ -105,7 +105,8 @@ def wrapped(payload, *args, **kwargs): 'source': source, 'destination': dest_parent, 'name': payload['destination']['name'], - 'is_check_permission': is_check_permission + 'is_check_permission': is_check_permission, + 'replaced_size': int(payload.get('replaced_size', 0)), }) except KeyError: raise HTTPError(http_status.HTTP_400_BAD_REQUEST) diff --git a/addons/osfstorage/tests/test_views_copy_move_quota.py b/addons/osfstorage/tests/test_views_copy_move_quota.py new file mode 100644 index 00000000000..cc166dc5666 --- /dev/null +++ b/addons/osfstorage/tests/test_views_copy_move_quota.py @@ -0,0 +1,865 @@ +# encoding: utf-8 +""" +Tests for new quota and FileInfo logic added to addons/osfstorage/views.py. + +Covers: + - _get_all_descendant_file_ids + - osfstorage_copy_hook (file + folder, FileInfo UPSERT, quota delta) + - osfstorage_move_hook (file + folder, intra-target replaced_size, inter-target quota) + - osfstorage_create_child (FileInfo UPSERT + quota delta on upload) + - osfstorage_delete (folder quota subtraction before deletion) +""" +from __future__ import unicode_literals + +import mock +import pytest +from nose.tools import assert_equal, assert_true, assert_false + +from framework.auth import signing +from website.util import api_url_for + +from addons.osfstorage.tests.utils import StorageTestCase, make_payload +from addons.osfstorage.tests import factories +from addons.osfstorage.tests.utils import recursively_create_file + +from osf.models import FileInfo, BaseFileNode +from osf_tests.factories import ProjectFactory + + +# --------------------------------------------------------------------------- +# Helper +# --------------------------------------------------------------------------- + +def _sign(payload): + """Return a signed copy of *payload* using the default signer.""" + return signing.sign_data(signing.default_signer, payload) + + +# --------------------------------------------------------------------------- +# Test helper: _get_all_descendant_file_ids +# --------------------------------------------------------------------------- + +@pytest.mark.django_db +class TestGetAllDescendantFileIds(StorageTestCase): + """Unit-tests for the private helper _get_all_descendant_file_ids.""" + + def _call(self, folder_node): + from addons.osfstorage.views import _get_all_descendant_file_ids + return _get_all_descendant_file_ids(folder_node) + + def test_empty_folder_returns_empty_list(self): + """An empty root folder (no children) must return [].""" + root = self.node_settings.get_root() + result = self._call(root) + assert_equal(result, []) + + def test_flat_folder_with_files(self): + """Files directly under a folder must all be returned.""" + root = self.node_settings.get_root() + f1 = root.append_file('alpha.txt') + f2 = root.append_file('beta.txt') + result = self._call(root) + assert set(result) == {f1.id, f2.id} + + def test_nested_folders_return_all_files(self): + """ + Structure: + root/ + sub1/ + deep.txt + file.txt + Both files must be returned, folder IDs must NOT be included. + """ + root = self.node_settings.get_root() + sub1 = root.append_folder('sub1') + deep = sub1.append_file('deep.txt') + flat = root.append_file('file.txt') + result = self._call(root) + assert set(result) == {deep.id, flat.id} + + def test_trashed_files_excluded(self): + """TrashedFile/TrashedFolder must not appear in the result.""" + root = self.node_settings.get_root() + live = root.append_file('live.txt') + dead = root.append_file('dead.txt') + # Soft-delete the file + dead.delete(user=self.user) + result = self._call(root) + assert live.id in result + assert dead.id not in result + + def test_only_osfstorage_provider(self): + """Files from other providers must be excluded (provider filter).""" + root = self.node_settings.get_root() + # append_file creates osfstorage files by default – just verify the + # provider guard does not block those + f = root.append_file('normal.txt') + result = self._call(root) + assert f.id in result + + +# --------------------------------------------------------------------------- +# Helpers shared by copy/move hook tests +# --------------------------------------------------------------------------- + +class HookTestBase(StorageTestCase): + """Base class providing signed hook helpers used by copy/move tests.""" + + def _post_hook(self, view_name, guid, payload): + return self.app.post_json( + api_url_for(view_name, guid=guid, **_sign(payload)), + {}, + expect_errors=True, + ) + + def _send_hook(self, view_name, guid_target, payload, target_node=None): + """Send a signed POST to *view_name* and return the response.""" + target_node = target_node or self.node + return self.app.post_json( + api_url_for(view_name, guid=guid_target), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + +# --------------------------------------------------------------------------- +# osfstorage_copy_hook – file copy +# --------------------------------------------------------------------------- +@pytest.mark.enable_implicit_clean +@pytest.mark.django_db +class TestCopyHookFileQuota(StorageTestCase): + """ + Tests for the FILE branch of osfstorage_copy_hook. + + New logic (views.py lines 159-208): + • Creates / updates a FileInfo record for the cloned file. + • Computes delta = new_size - replaced_size and calls update_quota. + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + def _copy_file(self, source_file, dest_folder, dest_target=None, replaced_size=0): + dest_target = dest_target or self.node + payload = { + 'source': source_file._id, + 'target': self.root_node._id, + 'user': self.user._id, + 'destination': { + 'parent': dest_folder._id, + 'target': dest_target._id, + 'name': source_file.name, + }, + } + if replaced_size: + payload['replaced_size'] = str(replaced_size) + # signed payload must go in the request BODY, not URL params + return self.app.post_json( + api_url_for('osfstorage_copy_hook', guid=self.node._id), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_file_creates_fileinfo(self, mock_st, mock_uq): + """After a file copy, a FileInfo record must exist for the clone.""" + version = factories.FileVersionFactory(size=500) + src = self.root_node.append_file('src.txt') + src.add_version(version) + src.save() + + dest_folder = self.root_node.append_folder('dest') + resp = self._copy_file(src, dest_folder) + assert resp.status_code == 201, f'copy hook returned {resp.status_code}: {resp.json}' + + # Exactly one FileInfo must exist for the cloned file (not the source) + cloned = dest_folder.find_child_by_name('src.txt') + assert cloned is not None + fi = FileInfo.objects.filter(file=cloned).first() + assert fi is not None + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_file_quota_delta_positive(self, mock_st, mock_uq): + """ + When a file is copied WITHOUT a replaced file (replaced_size=0), + the full new_size is added to the quota. + """ + version = factories.FileVersionFactory(size=300) + src = self.root_node.append_file('data.bin') + src.add_version(version) + src.save() + + dest_folder = self.root_node.append_folder('out') + resp = self._copy_file(src, dest_folder, replaced_size=0) + assert resp.status_code == 201, f'copy hook returned {resp.status_code}: {resp.json}' + + # update_quota must have been called with add=True and the correct size + assert mock_uq.called + call_kwargs = mock_uq.call_args + # delta > 0 → add=True + assert call_kwargs[1].get('add', call_kwargs[0][-1]) is True + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_file_no_quota_update_when_delta_zero(self, mock_st, mock_uq): + """ + When new_size == replaced_size the delta is 0, so update_quota must + NOT be called (views.py line 203: if delta != 0). + """ + version = factories.FileVersionFactory(size=100) + src = self.root_node.append_file('equal.txt') + src.add_version(version) + src.save() + + dest_folder = self.root_node.append_folder('same') + self._copy_file(src, dest_folder, replaced_size=100) + + assert not mock_uq.called + + +# --------------------------------------------------------------------------- +# osfstorage_copy_hook – folder copy +# --------------------------------------------------------------------------- + +@pytest.mark.enable_implicit_clean +@pytest.mark.django_db +class TestCopyHookFolderQuota(StorageTestCase): + """ + Tests for the FOLDER branch of osfstorage_copy_hook. + + New logic (views.py lines 165-198): + • Builds a source_size_map from FileInfo records (or falls back to + versions) for every descendant. + • Uses bulk_create (with IntegrityError fallback) to create FileInfo + records for the cloned folder's descendants. + • Forces replaced_size = 0 because osfstorage_delete has already + subtracted it. + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + def _copy_folder(self, source_folder, dest_folder, dest_target=None): + dest_target = dest_target or self.node + payload = { + 'source': source_folder._id, + 'target': self.root_node._id, + 'user': self.user._id, + 'destination': { + 'parent': dest_folder._id, + 'target': dest_target._id, + 'name': source_folder.name, + }, + } + # signed payload must go in the request BODY, not URL params + return self.app.post_json( + api_url_for('osfstorage_copy_hook', guid=self.node._id), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_folder_creates_fileinfo_for_descendants(self, mock_st, mock_uq): + """ + After copying a folder, every descendant FILE of the clone must have + a corresponding FileInfo record. + """ + # Build source folder with two files + src_folder = self.root_node.append_folder('src_folder') + v1 = factories.FileVersionFactory(size=200) + f1 = src_folder.append_file('one.txt') + f1.add_version(v1); f1.save() + + v2 = factories.FileVersionFactory(size=400) + f2 = src_folder.append_file('two.txt') + f2.add_version(v2); f2.save() + + # Pre-create FileInfo so source_size_map is populated + FileInfo.objects.update_or_create(file=f1, defaults={'file_size': 200}) + FileInfo.objects.update_or_create(file=f2, defaults={'file_size': 400}) + + dest_folder = self.root_node.append_folder('dest_folder') + resp = self._copy_folder(src_folder, dest_folder) + assert resp.status_code == 201, f'copy hook returned {resp.status_code}: {resp.json}' + + cloned = dest_folder.find_child_by_name('src_folder') + assert cloned is not None + + from addons.osfstorage.views import _get_all_descendant_file_ids + cloned_ids = _get_all_descendant_file_ids(cloned) + assert len(cloned_ids) == 2 + for fid in cloned_ids: + assert FileInfo.objects.filter(file_id=fid).exists(), \ + f'FileInfo missing for cloned file id={fid}' + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_folder_fallback_when_fileinfo_missing(self, mock_st, mock_uq): + """ + If the source file has no FileInfo record, the code falls back to + the latest FileVersion.size. The clone must still get a FileInfo + record (possibly size=0 if no version exists either). + """ + src_folder = self.root_node.append_folder('no_fi_folder') + v = factories.FileVersionFactory(size=111) + f = src_folder.append_file('nofi.txt') + f.add_version(v); f.save() + # Intentionally do NOT create a FileInfo for f + + dest_folder = self.root_node.append_folder('dest_nofi') + self._copy_folder(src_folder, dest_folder) + + cloned = dest_folder.find_child_by_name('no_fi_folder') + from addons.osfstorage.views import _get_all_descendant_file_ids + cloned_ids = _get_all_descendant_file_ids(cloned) + # The clone must have a FileInfo record + for fid in cloned_ids: + assert FileInfo.objects.filter(file_id=fid).exists() + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.osfstorage.views._get_all_descendant_file_ids') + def test_copy_folder_none_copied_from_id_sets_size_zero(self, mock_descendants, mock_st, mock_uq): + """ + When a cloned file has copied_from_id=None (not in source_size_map), + the else branch (views.py: 'src_id is None and not in map') sets size=0 + and still creates a FileInfo record for that file. + """ + src_folder = self.root_node.append_folder('src_none_id') + v = factories.FileVersionFactory(size=100) + src_file = src_folder.append_file('src_none.txt') + src_file.add_version(v) + src_file.save() + FileInfo.objects.update_or_create(file=src_file, defaults={'file_size': 100}) + + dest_folder = self.root_node.append_folder('dest_none_id') + # A fresh file with no copied_from → copied_from_id will be None in the DB + orphan_file = dest_folder.append_file('orphan.txt') + orphan_file.save() + + # First call → source descendants; second call → cloned descendants (orphan) + mock_descendants.side_effect = [ + [src_file.id], # _get_all_descendant_file_ids(source) + [orphan_file.id], # _get_all_descendant_file_ids(cloned) + ] + + resp = self._copy_folder(src_folder, dest_folder) + assert resp.status_code == 201 + + # The else branch must have produced size=0 → FileInfo.file_size == 0 + fi = FileInfo.objects.filter(file=orphan_file).first() + assert fi is not None, 'FileInfo must be created even when copied_from_id is None' + assert fi.file_size == 0 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_folder_bulk_create_integrity_error_fallback(self, mock_st, mock_uq): + """ + When FileInfo.objects.bulk_create raises IntegrityError the except-block + must call get_or_create for every FileInfo in file_infos. + """ + from django.db import IntegrityError as DjangoIntegrityError + + src_folder = self.root_node.append_folder('src_ie_folder') + v = factories.FileVersionFactory(size=75) + src_file = src_folder.append_file('ie_file.txt') + src_file.add_version(v) + src_file.save() + FileInfo.objects.update_or_create(file=src_file, defaults={'file_size': 75}) + + dest_folder = self.root_node.append_folder('dest_ie_folder') + + original_get_or_create = FileInfo.objects.get_or_create + with mock.patch.object( + FileInfo.objects, 'bulk_create', + side_effect=DjangoIntegrityError('duplicate key') + ): + with mock.patch.object( + FileInfo.objects, 'get_or_create', + wraps=original_get_or_create + ) as spy_goc: + resp = self._copy_folder(src_folder, dest_folder) + + assert resp.status_code == 201 + assert spy_goc.called, ( + 'get_or_create must be called as the IntegrityError fallback' + ) + # One get_or_create call per cloned descendant file + from addons.osfstorage.views import _get_all_descendant_file_ids + cloned = dest_folder.find_child_by_name('src_ie_folder') + cloned_ids = _get_all_descendant_file_ids(cloned) + assert spy_goc.call_count == len(cloned_ids) + + +# --------------------------------------------------------------------------- +# osfstorage_move_hook – quota logic +# --------------------------------------------------------------------------- + +@pytest.mark.django_db +class TestMoveHookQuota(StorageTestCase): + """ + Tests for the quota-related additions in osfstorage_move_hook. + + New logic (views.py lines 231-258): + FILE move: + • FileInfo UPSERT (file=source, file_size=latest_version.size). + FOLDER move: + • Aggregates FileInfo for all descendants. + • replaced_size = 0 (osfstorage_delete already subtracted). + QUOTA UPDATE: + • inter-target: subtract from source, add delta to dest. + • intra-target with replaced_size > 0: subtract replaced_size only. + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + def _move(self, source, dest_folder, dest_target=None, replaced_size=None, + is_check_permission=None): + dest_target = dest_target or self.node + payload = { + 'source': source._id, + 'target': self.root_node._id, + 'user': self.user._id, + 'destination': { + 'parent': dest_folder._id, + 'target': dest_target._id, + 'name': source.name, + }, + } + if replaced_size is not None: + payload['replaced_size'] = str(replaced_size) + if is_check_permission is not None: + payload['is_check_permission'] = is_check_permission + # signed payload must go in the request BODY, not URL params + return self.app.post_json( + api_url_for('osfstorage_move_hook', guid=self.node._id), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + # --- FILE MOVE --- + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_file_upserts_fileinfo(self, mock_st, mock_uq): + """ + Moving a FILE must create/update a FileInfo record for that file + reflecting the latest version size. + """ + version = factories.FileVersionFactory(size=250) + src = self.root_node.append_file('move_me.txt') + src.add_version(version); src.save() + + dest_folder = self.root_node.append_folder('landing') + self._move(src, dest_folder) + + src.reload() + fi = FileInfo.objects.filter(file=src).first() + assert fi is not None + assert fi.file_size == 250 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_file_inter_target_subtracts_source_quota(self, mock_st, mock_uq): + """ + Moving a FILE to a *different* target node must call update_quota + on the source target with add=False. + """ + other_project = ProjectFactory(creator=self.user) + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('remote') + + version = factories.FileVersionFactory(size=150) + src = self.root_node.append_file('cross.txt') + src.add_version(version); src.save() + + self._move(src, dest_folder, dest_target=other_project) + + # At least one call to update_quota with add=False (source deduction) + assert mock_uq.called + subtract_calls = [c for c in mock_uq.call_args_list + if c[1].get('add', c[0][-1]) is False] + assert len(subtract_calls) >= 1 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_file_intra_target_replaced_size_subtracted(self, mock_st, mock_uq): + """ + Moving a FILE within the SAME target node and providing replaced_size > 0 + must subtract replaced_size from the target (views.py line 255-257). + """ + version = factories.FileVersionFactory(size=100) + src = self.root_node.append_file('replace_target.txt') + src.add_version(version); src.save() + + dest_folder = self.root_node.append_folder('same_target_dest') + self._move(src, dest_folder, replaced_size=80) + + # update_quota must have been called (for the replace deduction) + assert mock_uq.called + subtract_calls = [c for c in mock_uq.call_args_list + if c[1].get('add', c[0][-1]) is False] + assert len(subtract_calls) >= 1 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_file_intra_target_no_replace_no_quota_update(self, mock_st, mock_uq): + """ + Moving a FILE within the SAME target and replaced_size=0 must NOT + trigger any update_quota call. + """ + version = factories.FileVersionFactory(size=100) + src = self.root_node.append_file('same_no_replace.txt') + src.add_version(version); src.save() + + dest_folder = self.root_node.append_folder('inner') + self._move(src, dest_folder, replaced_size=0) + + assert not mock_uq.called + + # --- FOLDER MOVE --- + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_folder_aggregates_fileinfo(self, mock_st, mock_uq): + """ + Moving a FOLDER must aggregate FileInfo sizes for all descendants + (new_size) and pass that to update_quota on the source (add=False). + """ + other_project = ProjectFactory(creator=self.user) + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('folder_dest') + + src_folder = self.root_node.append_folder('move_folder') + v1 = factories.FileVersionFactory(size=100) + f1 = src_folder.append_file('a.txt'); f1.add_version(v1); f1.save() + v2 = factories.FileVersionFactory(size=200) + f2 = src_folder.append_file('b.txt'); f2.add_version(v2); f2.save() + + FileInfo.objects.update_or_create(file=f1, defaults={'file_size': 100}) + FileInfo.objects.update_or_create(file=f2, defaults={'file_size': 200}) + + self._move(src_folder, dest_folder, dest_target=other_project) + + # update_quota must have been called for source deduction (300 total) + assert mock_uq.called + subtract_calls = [c for c in mock_uq.call_args_list + if c[1].get('add', c[0][-1]) is False] + assert len(subtract_calls) >= 1 + # The size passed must match the aggregate (300) + sizes = [c[0][1] for c in subtract_calls] + assert 300 in sizes + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_folder_replaced_size_zeroed(self, mock_st, mock_uq): + """ + When moving a folder, replaced_size must be set to 0 before computing + delta (osfstorage_delete already subtracted it). Therefore, if the + caller passes a non-zero replaced_size for a folder move the effective + delta equals new_size, NOT new_size - replaced_size. + """ + other_project = ProjectFactory(creator=self.user) + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('folder_dest2') + + src_folder = self.root_node.append_folder('move_folder2') + v = factories.FileVersionFactory(size=300) + f = src_folder.append_file('c.txt'); f.add_version(v); f.save() + FileInfo.objects.update_or_create(file=f, defaults={'file_size': 300}) + + # Pass a non-zero replaced_size – it must be ignored for folders + self._move(src_folder, dest_folder, dest_target=other_project, replaced_size=999) + + assert mock_uq.called + # The delta added to dest must be 300 (not 300-999=-699) + add_calls = [c for c in mock_uq.call_args_list + if c[1].get('add', c[0][-1]) is True] + sizes_added = [c[0][1] for c in add_calls] + assert 300 in sizes_added + + +# --------------------------------------------------------------------------- +# osfstorage_create_child – FileInfo UPSERT + quota delta +# --------------------------------------------------------------------------- + +@pytest.mark.django_db +class TestCreateChildQuota(StorageTestCase): + """ + Tests for the FileInfo / quota logic added to osfstorage_create_child. + + New logic (views.py lines 489-504): + • Computes new_size from the new FileVersion. + • Reads old_size from existing FileInfo (or 0 if none). + • UPSERT FileInfo with new_size. + • Calls update_quota(target, abs(delta), storage_type, add=(delta > 0)) + only when delta != 0. + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + def _upload(self, parent, name, size=123, unique_hash=None, target=None): + """ + Build and POST an upload payload. + + *unique_hash*: when provided, overrides ``metadata['name']`` so that + each call produces a distinct location-hash and is NOT treated as a + duplicate by ``FileVersion.is_duplicate``. Pass a different string on + each call to simulate a genuine new version. + """ + target = target or self.project + payload = make_payload(user=self.user, name=name) + payload['metadata']['size'] = size + if unique_hash is not None: + # Changing metadata['name'] changes the location['object'] value + # which changes the location_hash → not a duplicate + payload['metadata']['name'] = unique_hash + return self.app.post_json( + api_url_for( + 'osfstorage_create_child', + fid=parent._id, + guid=target._id, + ), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_upload_new_file_creates_fileinfo(self, mock_st, mock_uq): + """First upload of a file must create a FileInfo record with the correct size.""" + res = self._upload(self.root_node, 'brand_new.txt', size=500) + assert res.status_code in (200, 201) + record = self.root_node.find_child_by_name('brand_new.txt') + fi = FileInfo.objects.filter(file=record).first() + assert fi is not None + assert fi.file_size == 500 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_upload_new_file_calls_update_quota(self, mock_st, mock_uq): + """First upload (old_size=0) must call update_quota with add=True.""" + res = self._upload(self.root_node, 'new_quota.txt', size=300) + assert res.status_code in (200, 201) + assert mock_uq.called + call_kwargs = mock_uq.call_args + assert call_kwargs[1].get('add', call_kwargs[0][-1]) is True + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_upload_update_file_updates_fileinfo(self, mock_st, mock_uq): + """ + Re-uploading (new version) of the same file must update the FileInfo + to the new size. + + We pass unique_hash on each call so that the two uploads produce + different location hashes and are NOT treated as duplicates. Without + this, create_version returns the existing version (size=100) even when + we request size=200. + """ + # First upload + self._upload(self.root_node, 'versioned.txt', size=100, unique_hash='hash_v1') + record = self.root_node.find_child_by_name('versioned.txt') + + # Second upload – new hash forces a real new version with size=200 + mock_uq.reset_mock() + self._upload(self.root_node, 'versioned.txt', size=200, unique_hash='hash_v2') + record.reload() + fi = FileInfo.objects.filter(file=record).first() + assert fi is not None + assert fi.file_size == 200 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_upload_update_file_quota_delta(self, mock_st, mock_uq): + """ + When a file is updated with a genuinely new version, delta = new_size - old_size. + update_quota must be called with abs(delta) and add=True when size increases. + """ + # First upload sets old_size = 100 + self._upload(self.root_node, 'delta_test.txt', size=100, unique_hash='hash_a') + mock_uq.reset_mock() + + # Second upload: unique hash → real new version, new_size=250, delta=150 + self._upload(self.root_node, 'delta_test.txt', size=250, unique_hash='hash_b') + + assert mock_uq.called + call_kwargs = mock_uq.call_args + size_arg = call_kwargs[0][1] + add_arg = call_kwargs[1].get('add', call_kwargs[0][-1]) + assert size_arg == 150 + assert add_arg is True + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_upload_same_size_no_quota_update(self, mock_st, mock_uq): + """ + If the new version has the same size as the old version, delta=0 and + update_quota must NOT be called a second time. + """ + self._upload(self.root_node, 'same_size.txt', size=100) + mock_uq.reset_mock() + + # Upload again with the same size + self._upload(self.root_node, 'same_size.txt', size=100) + + # delta == 0 → update_quota should NOT be called + assert not mock_uq.called + + +# --------------------------------------------------------------------------- +# osfstorage_delete – folder quota subtraction +# --------------------------------------------------------------------------- + +@pytest.mark.django_db +class TestDeleteHookFolderQuota(StorageTestCase): + """ + Tests for the folder-quota logic added to osfstorage_delete. + + New logic (views.py lines 533-542): + • Collects _get_all_descendant_file_ids for the folder. + • Sums FileInfo.file_size for those files. + • Zeroes the FileInfo records BEFORE deletion (race-condition guard). + • Calls update_quota(target, total_size, storage_type, add=False). + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + def _delete(self, file_node): + payload = {'user': self.user._id} + return self.app.delete( + '{url}?payload={payload}&signature={signature}'.format( + url=api_url_for( + 'osfstorage_delete', + guid=self.node._id, + fid=file_node._id, + ), + **signing.sign_data(signing.default_signer, payload) + ), + expect_errors=True, + ) + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_delete_folder_subtracts_quota(self, mock_st, mock_uq): + """ + Deleting a folder with 2 files (total=300 bytes) must call + update_quota with size=300 and add=False. + """ + folder = self.root_node.append_folder('big_folder') + v1 = factories.FileVersionFactory(size=100) + f1 = folder.append_file('x.txt'); f1.add_version(v1); f1.save() + v2 = factories.FileVersionFactory(size=200) + f2 = folder.append_file('y.txt'); f2.add_version(v2); f2.save() + + FileInfo.objects.update_or_create(file=f1, defaults={'file_size': 100}) + FileInfo.objects.update_or_create(file=f2, defaults={'file_size': 200}) + + resp = self._delete(folder) + assert resp.status_code == 200 + + assert mock_uq.called + call_args = mock_uq.call_args + size_arg = call_args[0][1] + add_arg = call_args[1].get('add', call_args[0][-1]) + assert size_arg == 300 + assert add_arg is False + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_delete_folder_zeroes_fileinfo_before_deletion(self, mock_st, mock_uq): + """ + The FileInfo records must be zeroed BEFORE the hard deletion of the + folder to prevent race conditions (views.py line 541). + """ + folder = self.root_node.append_folder('zeroed_folder') + v = factories.FileVersionFactory(size=500) + f = folder.append_file('z.txt'); f.add_version(v); f.save() + FileInfo.objects.update_or_create(file=f, defaults={'file_size': 500}) + + # We need to capture the FileInfo state at the moment update_quota is + # called – at that point file_size must already be 0. + recorded_file_size = [] + + def side_effect(target, size, storage_type, add): + fi = FileInfo.objects.filter(file=f).first() + if fi: + recorded_file_size.append(fi.file_size) + + mock_uq.side_effect = side_effect + + self._delete(folder) + + assert len(recorded_file_size) == 1 + assert recorded_file_size[0] == 0, ( + 'FileInfo.file_size must be 0 when update_quota is called, ' + f'but was {recorded_file_size[0]}' + ) + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_delete_empty_folder_no_quota_update(self, mock_st, mock_uq): + """ + Deleting an empty folder (total_size=0) must NOT call update_quota + (views.py line 538: if total_size > 0). + """ + folder = self.root_node.append_folder('empty_folder') + resp = self._delete(folder) + assert resp.status_code == 200 + assert not mock_uq.called + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_delete_folder_nested_files_quota(self, mock_st, mock_uq): + """ + Nested files (folder-in-folder) must ALL be counted when computing + the quota deduction. + """ + outer = self.root_node.append_folder('outer') + inner = outer.append_folder('inner') + + v1 = factories.FileVersionFactory(size=50) + f1 = inner.append_file('deep1.txt'); f1.add_version(v1); f1.save() + v2 = factories.FileVersionFactory(size=150) + f2 = inner.append_file('deep2.txt'); f2.add_version(v2); f2.save() + + FileInfo.objects.update_or_create(file=f1, defaults={'file_size': 50}) + FileInfo.objects.update_or_create(file=f2, defaults={'file_size': 150}) + + resp = self._delete(outer) + assert resp.status_code == 200 + + assert mock_uq.called + size_arg = mock_uq.call_args[0][1] + assert size_arg == 200 # 50 + 150 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_delete_file_does_not_trigger_folder_quota_path(self, mock_st, mock_uq): + """ + Deleting a single FILE must NOT go through the folder-quota path. + The new code block is guarded by `not file_node.is_file`. + """ + v = factories.FileVersionFactory(size=777) + f = self.root_node.append_file('single.txt') + f.add_version(v); f.save() + FileInfo.objects.update_or_create(file=f, defaults={'file_size': 777}) + + resp = self._delete(f) + assert resp.status_code == 200 + + # update_quota must NOT have been called by the folder-quota path. + # (The existing file-quota path in views.py does NOT call update_quota + # on delete; that is handled by the WaterButler hook elsewhere.) + assert not mock_uq.called diff --git a/addons/osfstorage/views.py b/addons/osfstorage/views.py index 1e413f84df2..ef37de8d090 100644 --- a/addons/osfstorage/views.py +++ b/addons/osfstorage/views.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from django.db.models import IntegerField +from django.db.models import IntegerField, Sum from django.db.models.functions import Cast from rest_framework import status as http_status import logging @@ -19,7 +19,7 @@ from api.caching.tasks import update_storage_usage from osf.exceptions import InvalidTagError, TagNotFoundError -from osf.models import FileVersion, OSFUser, ExportDataRestore, ExportData +from osf.models import FileVersion, OSFUser, ExportDataRestore, ExportData, FileInfo, BaseFileNode from osf.utils.permissions import WRITE from osf.utils.requests import check_select_for_update from website.project.decorators import ( @@ -28,6 +28,7 @@ from website.project.model import has_anonymous_link from website.files import exceptions +from website.util.quota import get_project_storage_type, update_quota from addons.osfstorage import utils from addons.osfstorage import decorators from addons.osfstorage.models import OsfStorageFolder @@ -124,17 +125,94 @@ def osfstorage_get_revisions(file_node, payload, target, **kwargs): ] } +def _get_all_descendant_file_ids(folder_node): + TRASHED_TYPES = ('osf.trashedfilenode', 'osf.trashedfile', 'osf.trashedfolder') + + file_ids = [] + folder_ids_to_process = [folder_node.id] + + while folder_ids_to_process: + children = BaseFileNode.objects.filter( + parent_id__in=folder_ids_to_process, + provider='osfstorage', + ).exclude( + type__in=TRASHED_TYPES + ).values('id', 'type') + + next_folder_ids = [] + for child in children: + if child['type'] == 'osf.osfstoragefile': + file_ids.append(child['id']) + elif child['type'] == 'osf.osfstoragefolder': + next_folder_ids.append(child['id']) + + folder_ids_to_process = next_folder_ids + + return file_ids @decorators.waterbutler_opt_hook def osfstorage_copy_hook(source, destination, name=None, **kwargs): - ret = source.copy_under(destination, name=name).serialize(), http_status.HTTP_201_CREATED + replaced_size = int(kwargs.get('replaced_size', 0)) + cloned = source.copy_under(destination, name=name) + + # Only handle if source is a node and destination is a node + if getattr(source.target, 'type', '') == 'osf.node' and getattr(destination.target, 'type', '') == 'osf.node': + if source.is_file: + latest_version = cloned.versions.order_by('-created').first() + new_size = latest_version.size if latest_version and latest_version.size else 0 + FileInfo.objects.update_or_create(file=cloned, defaults={'file_size': new_size}) + else: + source_file_ids = _get_all_descendant_file_ids(source) + source_size_map = { + fi.file_id: fi.file_size + for fi in FileInfo.objects.filter(file_id__in=source_file_ids) + } + + cloned_file_ids = _get_all_descendant_file_ids(cloned) + cloned_files = list(BaseFileNode.objects.filter( + id__in=cloned_file_ids + ).values('id', 'copied_from_id')) + + file_infos = [] + new_size = 0 + for cf in cloned_files: + src_id = cf['copied_from_id'] + if src_id in source_size_map: + size = source_size_map[src_id] + elif src_id is not None: + # FileInfo not exist + node = BaseFileNode.objects.filter(id=src_id).first() + size = 0 + if node: + v = node.versions.order_by('-created').first() + size = (v.size or 0) if v else 0 + else: # src_id is None and not in map + size = 0 + + file_infos.append(FileInfo(file_id=cf['id'], file_size=size)) + new_size += size + + try: + FileInfo.objects.bulk_create(file_infos) + except IntegrityError: + for fi in file_infos: + FileInfo.objects.get_or_create(file_id=fi.file_id, defaults={'file_size': fi.file_size}) + replaced_size = 0 # osfstorage_delete already subtracted this + + # UserQuota: delta = new_size - replaced_size + delta = new_size - replaced_size + if delta != 0: + storage_type = get_project_storage_type(destination.target) + update_quota(destination.target, abs(delta), storage_type, add=(delta > 0)) + update_storage_usage(destination.target) - return ret + return cloned.serialize(), http_status.HTTP_201_CREATED @decorators.waterbutler_opt_hook def osfstorage_move_hook(source, destination, name=None, **kwargs): source_target = source.target is_check_permission = kwargs.get('is_check_permission') + replaced_size = int(kwargs.get('replaced_size', 0)) try: ret = source.move_under(destination, name=name, is_check_permission=is_check_permission).serialize(), http_status.HTTP_200_OK except exceptions.FileNodeCheckedOutError: @@ -150,6 +228,32 @@ def osfstorage_move_hook(source, destination, name=None, **kwargs): 'message_long': 'Cannot move file as it is the primary file of preprint.' }) + # Only handle if source is a node and destination is a node + if getattr(source_target, 'type', '') == 'osf.node' and getattr(destination.target, 'type', '') == 'osf.node': + if source.is_file: + latest_version = source.versions.order_by('-created').first() + new_size = latest_version.size if latest_version and latest_version.size else 0 + # FileInfo: UPSERT + FileInfo.objects.update_or_create(file=source, defaults={'file_size': new_size}) + else: + source_file_ids = _get_all_descendant_file_ids(source) # recursive + new_size = FileInfo.objects.filter( + file_id__in=source_file_ids + ).aggregate(total=Sum('file_size'))['total'] or 0 + replaced_size = 0 # osfstorage_delete already subtracted this + + # UserQuota + if source_target != destination.target: + src_storage = get_project_storage_type(source_target) + dest_storage = get_project_storage_type(destination.target) + update_quota(source_target, new_size, src_storage, add=False) + delta = new_size - replaced_size + if delta != 0: + update_quota(destination.target, abs(delta), dest_storage, add=(delta > 0)) + elif replaced_size > 0: + storage_type = get_project_storage_type(destination.target) + update_quota(destination.target, replaced_size, storage_type, add=False) + # once the move is complete recalculate storage for both targets if it's a inter-target move. if is_check_permission and source_target != destination.target: update_storage_usage(destination.target) @@ -379,6 +483,24 @@ def osfstorage_create_child(file_node, payload, **kwargs): if not current_version or not current_version.is_duplicate(new_version): update_storage_usage(file_node.target) + # Only handle if target is a node + if getattr(file_node.target, 'type', '') == 'osf.node': + new_size = new_version.size if new_version.size and new_version.size > 0 else 0 + if new_size >= 0: + old_info = FileInfo.objects.filter(file=file_node).first() + old_size = old_info.file_size if old_info else 0 + + # FileInfo: UPSERT + FileInfo.objects.update_or_create( + file=file_node, defaults={'file_size': new_size} + ) + + # UserQuota: delta = new - old + delta = new_size - old_size + if delta != 0: + storage_type = get_project_storage_type(file_node.target) + update_quota(file_node.target, abs(delta), storage_type, add=(delta > 0)) + version_id = new_version._id archive_exists = new_version.archive is not None else: @@ -406,6 +528,17 @@ def osfstorage_delete(file_node, payload, target, **kwargs): if file_node == OsfStorageFolder.objects.get_root(target=target): raise HTTPError(http_status.HTTP_400_BAD_REQUEST) + if getattr(file_node.target, 'type', '') == 'osf.node' and not file_node.is_file: + # Since we are deleting a folder, we need to calculate the total size of all descendant files to update quota + all_file_ids = _get_all_descendant_file_ids(file_node) + total_size = FileInfo.objects.filter(file_id__in=all_file_ids) \ + .aggregate(total=Sum('file_size'))['total'] or 0 + if total_size > 0: + storage_type = get_project_storage_type(target) + # Subtract quota BEFORE deletion; also zero FileInfo to avoid race conditions with new file creation during deletion + FileInfo.objects.filter(file_id__in=all_file_ids).update(file_size=0) + update_quota(target, total_size, storage_type, add=False) + try: file_node.delete(user=user) diff --git a/tests/test_addons.py b/tests/test_addons.py index e6f18031b56..da39ce95634 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -113,7 +113,9 @@ def test_auth_download(self): data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data'] assert_equal(data['auth'], views.make_auth(self.user)) assert_equal(data['credentials'], self.node_addon.serialize_waterbutler_credentials()) - assert_equal(data['settings'], self.node_addon.serialize_waterbutler_settings()) + expected_settings = self.node_addon.serialize_waterbutler_settings().copy() + expected_settings['max_file_size'] = 100 + assert_equal(data['settings'], expected_settings) expected_url = furl.furl(self.node.api_url_for('create_waterbutler_log', _absolute=True, _internal=True)) observed_url = furl.furl(data['callback_url']) observed_url.port = expected_url.port @@ -153,7 +155,9 @@ def test_auth_bad_cookie(self): data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data'] assert_equal(data['auth'], views.make_auth(self.user)) assert_equal(data['credentials'], self.node_addon.serialize_waterbutler_credentials()) - assert_equal(data['settings'], self.node_addon.serialize_waterbutler_settings()) + expected_settings = self.node_addon.serialize_waterbutler_settings().copy() + expected_settings['max_file_size'] = 100 + assert_equal(data['settings'], expected_settings) expected_url = furl.furl(self.node.api_url_for('create_waterbutler_log', _absolute=True, _internal=True)) observed_url = furl.furl(data['callback_url']) observed_url.port = expected_url.port diff --git a/tests/test_quota_move_copy.py b/tests/test_quota_move_copy.py new file mode 100644 index 00000000000..c39830d94e0 --- /dev/null +++ b/tests/test_quota_move_copy.py @@ -0,0 +1,248 @@ +import pytest +from unittest import mock +from website.util import quota +from django.db import IntegrityError + +@pytest.fixture +def mock_user(): + user = mock.Mock() + user.id = 1 + user._id = 'abc123' + user.userquota_set.get.side_effect = quota.UserQuota.DoesNotExist + return user + +@pytest.fixture +def mock_node(): + node = mock.Mock(spec=quota.AbstractNode) + node.id = 42 + node.creator = mock.Mock() + node.type = 'osf.node' + return node + +def test_get_file_size_file(): + children = {'children': [{'kind': 'file', 'size': 100}, {'kind': 'file', 'size': 200}]} + assert quota.get_file_size(children) == 300 + +def test_get_file_size_nested(): + children = { + 'children': [ + {'kind': 'file', 'size': 100}, + {'kind': 'folder', 'children': [{'kind': 'file', 'size': 50}]} + ] + } + assert quota.get_file_size(children) == 150 + +@mock.patch('website.util.quota.ProjectStorageType.objects.get') +@mock.patch('website.util.quota.AbstractNode.objects.get') +@mock.patch('website.util.quota.update_quota') +def test_handle_move_copy_dest_osfstorage(mock_update_quota, mock_node_get, mock_pst_get, mock_node): + # Mock ProjectStorageType.objects.get to return a mock with storage_type + mock_pst_get.return_value = mock.Mock(storage_type=1) + # Mock AbstractNode.objects.get to return a mock node + mock_node_get.return_value = mock_node + + payload = { + 'destination': {'provider': 's3compatinstitutions', 'size': 123, 'children': None}, + 'source': {'nid': 'nid1', 'provider': 'osfstorage'} + } + quota._handle_move_copy(quota.FileLog.FILE_MOVED, mock_node, mock.Mock(), payload) + mock_update_quota.assert_called_once() + +@mock.patch('website.util.quota.ProjectStorageType.objects.get') +@mock.patch('website.util.quota.AbstractNode.objects.get') +@mock.patch('website.util.quota.update_quota') +@pytest.mark.django_db +def test_handle_move_copy_source_osfstorage(mock_update_quota, mock_node_get, mock_pst_get, mock_user): + mock_node = mock.create_autospec(quota.AbstractNode, instance=True) + mock_node.type = 'osf.node' + mock_pst_get.return_value = mock.Mock(storage_type=1) + mock_node_get.return_value = mock_node + + payload = { + 'destination': {'provider': 's3compatinstitutions', 'size': 123}, # <-- thêm size + 'source': {'provider': 'osfstorage', 'size': 123, 'nid': 'nid1'} + } + quota._handle_move_copy(quota.FileLog.FILE_MOVED, mock_node, mock_user, payload) + mock_update_quota.assert_called_once() + +@mock.patch('website.util.quota.update_quota') +@pytest.mark.django_db +def test_handle_move_copy_no_osfstorage(mock_update_quota, mock_node, mock_user): + payload = { + 'destination': {'provider': 's3compatinstitutions'}, + 'source': {'provider': 's3compatinstitutions'} + } + quota._handle_move_copy(quota.FileLog.FILE_MOVED, mock_node, mock_user, payload) + mock_update_quota.assert_not_called() + +@mock.patch('website.util.quota.ProjectStorageType.objects.get') +@mock.patch('website.util.quota.AbstractNode.objects.get') +@mock.patch('website.util.quota.update_quota') +@pytest.mark.django_db +def test_file_moved_file(mock_update_quota, mock_node_get, mock_pst_get, mock_node): + mock_pst_get.return_value = mock.Mock(storage_type=1) + mock_node_get.return_value = mock_node + payload = { + 'destination': {'size': 123, 'children': None}, + 'source': {'nid': 'nid1', 'provider': 'osfstorage'} + } + quota.file_moved(mock_node, payload) + mock_update_quota.assert_called_once() + +@mock.patch('website.util.quota.ProjectStorageType.objects.get') +@mock.patch('website.util.quota.AbstractNode.objects.get') +@mock.patch('website.util.quota.update_quota') +@pytest.mark.django_db +def test_file_moved_folder(mock_update_quota, mock_node_get, mock_pst_get, mock_node): + mock_pst_get.return_value = mock.Mock(storage_type=1) + mock_node_get.return_value = mock_node + payload = { + 'destination': {'children': [{'kind': 'file', 'size': 100}]}, + 'source': {'nid': 'nid1', 'provider': 'osfstorage'} + } + quota.file_moved(mock_node, payload) + mock_update_quota.assert_called_once() + +@mock.patch('website.util.quota.ProjectStorageType.objects.get', side_effect=Exception) +@mock.patch('website.util.quota.AbstractNode.objects.get') +@mock.patch('website.util.quota.update_quota') +@pytest.mark.django_db +def test_file_moved_exception(mock_update_quota, mock_node_get, mock_pst_get, mock_node): + mock_node_get.return_value = mock_node + payload = { + 'destination': {'size': 123, 'children': None}, + 'source': {'nid': 'nid1', 'provider': 'osfstorage'} + } + with pytest.raises(Exception): + quota.file_moved(mock_node, payload) + mock_update_quota.assert_not_called() + +@mock.patch('website.util.quota.UserQuota.objects.get') +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_add(mock_filter, mock_get, mock_node): + uq = mock.Mock() + uq.used = 0 + mock_get.return_value = uq + mock_filter.return_value.select_for_update.return_value.get.return_value = uq + quota.update_quota(mock_node, 100, 1, add=True) + assert uq.used == 100 + +@mock.patch('website.util.quota.UserQuota.objects.get') +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_subtract(mock_filter, mock_get, mock_node): + uq = mock.Mock() + uq.used = 200 + mock_get.return_value = uq + mock_filter.return_value.select_for_update.return_value.get.return_value = uq + quota.update_quota(mock_node, 50, 1, add=False) + assert uq.used == 150 + +@mock.patch('website.util.quota.check_select_for_update', return_value=False) +@mock.patch('website.util.quota.transaction.atomic') +@mock.patch('website.util.quota.UserQuota.objects.create') +@mock.patch('website.util.quota.UserQuota.objects.filter') +@mock.patch('website.util.quota.UserQuota.objects.get', side_effect=quota.UserQuota.DoesNotExist) +@pytest.mark.django_db +def test_update_quota_userquota_not_exist(mock_get, mock_filter, mock_create, mock_atomic, mock_node): + mock_filter.return_value.select_for_update.return_value.get.side_effect = quota.UserQuota.DoesNotExist + quota.update_quota(mock_node, 50, 1, add=True) + mock_get.assert_called_once() + +@mock.patch('website.util.quota.UserQuota.objects.get', side_effect=Exception) +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_other_exception(mock_filter, mock_get, mock_node): + mock_filter.return_value.select_for_update.return_value.get.side_effect = Exception + with pytest.raises(Exception): + quota.update_quota(mock_node, 50, 1, add=True) + mock_get.assert_not_called() + +@mock.patch('website.util.quota._handle_move_copy') +def test_update_used_quota_move_event(mock_handle_move_copy, mock_node, mock_user): + payload = { + 'destination': {'provider': 's3compatinstitutions'}, + 'source': {'provider': 'osfstorage'} + } + quota.update_used_quota(None, mock_node, mock_user, quota.FileLog.FILE_MOVED, payload) + mock_handle_move_copy.assert_called_once() + +@mock.patch('website.util.quota._handle_move_copy') +def test_update_used_quota_dest_osfstorage_returns_early(mock_handle_move_copy, mock_node, mock_user): + payload = { + 'destination': {'provider': 'osfstorage'} + } + result = quota.update_used_quota(None, mock_node, mock_user, quota.FileLog.FILE_MOVED, payload) + assert result is None + mock_handle_move_copy.assert_not_called() + +@mock.patch('website.util.quota.check_select_for_update', return_value=True) +@mock.patch('website.util.quota.transaction.atomic') +@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError) +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_integrityerror_select_for_update_add( + mock_filter, mock_create, mock_atomic, mock_check, mock_node +): + uq = mock.Mock() + uq.used = 10 + mock_filter.return_value.select_for_update.return_value.get.side_effect = [ + quota.UserQuota.DoesNotExist, + uq + ] + quota.update_quota(mock_node, 5, 1, add=True) + assert mock_filter.return_value.select_for_update.return_value.get.call_count == 2 + assert uq.used == 15 + uq.save.assert_called_once() + +@mock.patch('website.util.quota.check_select_for_update', return_value=True) +@mock.patch('website.util.quota.transaction.atomic') +@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError) +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_integrityerror_select_for_update_subtract( + mock_filter, mock_create, mock_atomic, mock_check, mock_node +): + uq = mock.Mock() + uq.used = 10 + mock_filter.return_value.select_for_update.return_value.get.side_effect = [ + quota.UserQuota.DoesNotExist, + uq + ] + quota.update_quota(mock_node, 3, 1, add=False) + assert mock_filter.return_value.select_for_update.return_value.get.call_count == 2 + assert uq.used == 7 + uq.save.assert_called_once() + +@mock.patch('website.util.quota.check_select_for_update', return_value=False) +@mock.patch('website.util.quota.transaction.atomic') +@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError) +@mock.patch('website.util.quota.UserQuota.objects.get') +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_integrityerror_get_add( + mock_filter, mock_get, mock_create, mock_atomic, mock_check, mock_node +): + uq = mock.Mock() + uq.used = 20 + mock_get.side_effect = [quota.UserQuota.DoesNotExist, uq] + quota.update_quota(mock_node, 8, 1, add=True) + assert uq.used == 28 + uq.save.assert_called_once() + +@mock.patch('website.util.quota.check_select_for_update', return_value=False) +@mock.patch('website.util.quota.transaction.atomic') +@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError) +@mock.patch('website.util.quota.UserQuota.objects.get') +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_integrityerror_get_subtract( + mock_filter, mock_get, mock_create, mock_atomic, mock_check, mock_node +): + uq = mock.Mock() + uq.used = 30 + mock_get.side_effect = [quota.UserQuota.DoesNotExist, uq] + quota.update_quota(mock_node, 12, 1, add=False) + assert uq.used == 18 + uq.save.assert_called_once() diff --git a/website/static/js/fangorn.js b/website/static/js/fangorn.js index 2333b25ec93..0cb301ea584 100644 --- a/website/static/js/fangorn.js +++ b/website/static/js/fangorn.js @@ -583,6 +583,25 @@ function checkConflictsRename(tb, item, name, cb) { cb('replace'); } +/** + * Recursively finds all individual files that exceed maxSizeBytes. + * Returns an array of offending item objects. + */ +function _findOversizedFiles(item, maxSizeBytes) { + var oversized = []; + if (item.kind === 'file') { + var size = item.data.size || 0; + if (size > 0 && size > maxSizeBytes) { + oversized.push(item); + } + } else if (item.kind === 'folder' && item.children && item.children.length > 0) { + for (var i = 0; i < item.children.length; i++) { + oversized = oversized.concat(_findOversizedFiles(item.children[i], maxSizeBytes)); + } + } + return oversized; +} + function doItemOp(operation, to, from, rename, conflict) { var tb = this; // dismiss old modal immediately to prevent button mashing @@ -626,6 +645,75 @@ function doItemOp(operation, to, from, rename, conflict) { return; } + if (operation !== OPERATIONS.RENAME) { + var destMaxSize = to.data && to.data.accept && to.data.accept.maxSize; + if (destMaxSize) { + var maxSizeBytes = destMaxSize * 1000000; + var oversizedFiles = _findOversizedFiles(from, maxSizeBytes); + if (oversizedFiles.length > 0) { + var maxSizeDisplay = $osf.humanFileSize(maxSizeBytes, true); + oversizedFiles.forEach(function(oversized) { + var displaySize = $osf.humanFileSize(oversized.data.size, true); + $osf.growl(sprintf( + gettext('File「%1$s」is too large (%2$s). Max file size is %3$s.'), + oversized.data.name, displaySize, maxSizeDisplay + )); + }); + from.inProgress = false; + return; + } + } + + var destProvider = to.data.provider; + if (destProvider === 'osfstorage') { + // Sum size of all files in the item being moved/copied + var totalMoveSize = 0; + if (from.kind === 'file') { + totalMoveSize = from.data.size || 0; + } else { + getAllChildren(from).forEach(function(child) { + if (child.kind === 'file') { + totalMoveSize += child.data.size || 0; + } + }); + } + + if (totalMoveSize > 0) { + var quotaResp = $.ajax({ + async: false, + method: 'GET', + url: to.data.nodeApiUrl + 'get_creator_quota/' + }); + if (quotaResp.responseJSON) { + var quotaData = quotaResp.responseJSON; + var quotaMsgText = ''; + + if (quotaData.used + totalMoveSize > quotaData.max) { + if (from.kind === 'file') { + quotaMsgText = sprintf(gettext('Not enough quota to move/copy the file.')); + } else { + var folderSize = $osf.humanFileSize(totalMoveSize, true); + quotaMsgText = sprintf(gettext('Not enough quota to move/copy. The total size of the folder %1$s.'), folderSize); + } + from.notify.update(quotaMsgText, 'warning', undefined, 3000); + from.inProgress = false; + return; + } + if (quotaData.used + totalMoveSize > quotaData.max * window.contextVars.threshold) { + $osf.growl( + gettext('Quota usage alert'), + sprintf( + gettext('You have used more than %1$s%% of your quota.'), + (window.contextVars.threshold * 100) + ), + 'warning' + ); + } + } + } + } + } + var origFrom = Object.assign({}, from); if (operation === OPERATIONS.COPY) { from = tb.createItem($.extend(true, {status: operation.status}, from.data), to.id); @@ -731,6 +819,39 @@ function doItemOp(operation, to, from, rename, conflict) { from.data.status = undefined; } + if (xhr.status === 413 && xhr.responseJSON && xhr.responseJSON.oversized_files) { + var destMaxSize = to.data && to.data.accept && to.data.accept.maxSize; + var maxSizeDisplay = destMaxSize ? $osf.humanFileSize(destMaxSize * 1000000, true) : '5 GB'; + + xhr.responseJSON.oversized_files.forEach(function(fileObj) { + var displaySize = $osf.humanFileSize(fileObj.size, true); + $osf.growl(sprintf( + gettext('File「%1$s」is too large (%2$s). Max file size is %3$s.'), + fileObj.name, displaySize, maxSizeDisplay + )); + }); + + if (notRenameOp) { + addFileStatus(tb, from, false, '', '', conflict); + } + orderFolder.call(tb, from.parent()); + tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function (file) { return file.data.id !== from.data.id; }); + from.inProgress = false; + return; + } + if (xhr.status === 406 && xhr.responseJSON && xhr.responseJSON.message) { + $osf.growl(sprintf( + gettext(xhr.responseJSON.message) + )); + if (notRenameOp) { + addFileStatus(tb, from, false, '', '', conflict); + } + orderFolder.call(tb, from.parent()); + tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function (file) { return file.data.id !== from.data.id; }); + from.inProgress = false; + return; + } + var message; if (xhr.status !== 500 && xhr.responseJSON && (xhr.responseJSON.message || xhr.responseJSON.message_long)) { @@ -3361,6 +3482,7 @@ function _dropLogic(event, items, folder) { if (toMove.conflicts.length > 0) { tb.syncFileMoveCache[folder.data.provider].conflicts = tb.syncFileMoveCache[folder.data.provider].conflicts || []; + tb.syncFileMoveCache[folder.data.provider].ready = tb.syncFileMoveCache[folder.data.provider].ready || []; toMove.conflicts.forEach(function(item) { tb.syncFileMoveCache[folder.data.provider].conflicts.push({'item' : item, 'folder' : folder}); }); diff --git a/website/translations/en/LC_MESSAGES/js_messages.po b/website/translations/en/LC_MESSAGES/js_messages.po index 674d3c0fa1d..07195312429 100644 --- a/website/translations/en/LC_MESSAGES/js_messages.po +++ b/website/translations/en/LC_MESSAGES/js_messages.po @@ -9288,3 +9288,15 @@ msgstr "" msgid "The Integrated Admin added ${contributors} as contributor(s) to ${node}" msgstr "" + +msgid "File「%1$s」is too large (%2$s). Max file size is %3$s." +msgstr "" + +msgid "You do not have enough available quota." +msgstr "" + +msgid "Not enough quota to move/copy the file." +msgstr "" + +msgid "Not enough quota to move/copy. The total size of the folder %1$s." +msgstr "" diff --git a/website/translations/ja/LC_MESSAGES/js_messages.po b/website/translations/ja/LC_MESSAGES/js_messages.po index 81dd0b3e5e0..a4e897f9f26 100644 --- a/website/translations/ja/LC_MESSAGES/js_messages.po +++ b/website/translations/ja/LC_MESSAGES/js_messages.po @@ -10841,3 +10841,15 @@ msgstr "" "
  • このアドオンにより、GakuNin RDMプロジェクトは外部サービスに接続されます。このサービスを利用すると、外部サービスの利用規約に拘束されます。GakuNin RDMはサービスおよびその利用について責任を負いません。
    • \n" "
  • このアドオンを利用すると外部サービスにファイルを保存できます。このアドオンに追加したファイルはGakuNin RDM内には保存されません。
    • \n" "\n" + +msgid "File「%1$s」is too large (%2$s). Max file size is %3$s." +msgstr "ファイル「%1$s」は大きすぎます (%2$s)。最大ファイルサイズは %3$s です。" + +msgid "You do not have enough available quota." +msgstr "利用可能なクォータが不足しています。" + +msgid "Not enough quota to move/copy the file." +msgstr "ファイルを移動/コピーするのに十分なクォータがありません。" + +msgid "Not enough quota to move/copy. The total size of the folder %1$s." +msgstr "移動/コピーするための空き容量が足りません。フォルダのサイズは%1$sです。" diff --git a/website/translations/js_messages.pot b/website/translations/js_messages.pot index 7086b672e57..ecaba5d4cda 100644 --- a/website/translations/js_messages.pot +++ b/website/translations/js_messages.pot @@ -9241,3 +9241,15 @@ msgstr "" msgid "The Integrated Admin added ${contributors} as contributor(s) to ${node}" msgstr "" + +msgid "File「%1$s」 is too large (%2$s). Max file size is %3$s." +msgstr "" + +msgid "You do not have enough available quota." +msgstr "" + +msgid "Not enough quota to move/copy the file." +msgstr "" + +msgid "Not enough quota to move/copy. The total size of the folder %1$s." +msgstr "" diff --git a/website/util/quota.py b/website/util/quota.py index 1990073a444..ed22033b718 100644 --- a/website/util/quota.py +++ b/website/util/quota.py @@ -142,6 +142,17 @@ def get_project_storage_type(node): @file_signals.file_updated.connect def update_used_quota(self, target, user, event_type, payload): + # FILE_MOVED and FILE_COPIED carry 'source'/'destination', not 'metadata'. + # Handle them first before the metadata_provider guard below. + if event_type in (FileLog.FILE_MOVED, FileLog.FILE_COPIED): + dest_provider = payload.get('destination', {}).get('provider') + if dest_provider == 'osfstorage': + return # Hooks already handled (create_child), SKIP double counting + src_provider = payload.get('source', {}).get('provider') + if src_provider == 'osfstorage' or src_provider in PROVIDERS: + _handle_move_copy(event_type, target, user, payload) + return + data = dict(payload.get('metadata')) if payload.get('metadata') else None metadata_provider = data.get('provider') if payload.get('metadata') else None if metadata_provider == 'osfstorage' or metadata_provider in PROVIDERS: @@ -390,3 +401,105 @@ def get_node_file_list(file_node): file_list.append(child_file_node) return file_list + +def _handle_move_copy(event_type, target, user, payload): + """Resolve the destination file_node and dispatch to file_moved or file_copied. + + For FILE_MOVED / FILE_COPIED events the relevant file information lives in + ``payload['destination']``, not in ``payload['metadata']``. Only extended-storage + providers (PROVIDERS) destinations are processed here. + + Scenarios handled: + - osfstorage → extended storage (quota decreases for source node) + - same provider moves / copies (if it involves extended storage) + + (Note: Moves TO osfstorage are handled entirely by OSF hooks, bypassing this flow). + + :param str event_type: FileLog.FILE_MOVED or FileLog.FILE_COPIED + :param AbstractNode target: destination project node + :param OSFUser user: acting user + :param dict payload: full webhook payload + """ + + if event_type == FileLog.FILE_MOVED: + file_moved(target, payload) + +def file_moved(target, payload): + """Update per-user-per-storage used quota when moving file + + :param Object target: Id of project + :param dict payload: The payload of the move operation + """ + if isinstance(target, AbstractNode): + file_size = -1 + dest_size = payload['destination'].get('size', None) + children = payload['destination'].get('children', None) + # Move file + if dest_size is not None: + file_size = int(dest_size) + # Move folder + elif children is not None: + file_size = get_file_size(payload['destination']) + if file_size < 0: + return + + source_node_id = payload['source']['nid'] + source_node = AbstractNode.objects.get(guids___id=source_node_id) + # Move from bulk-mount + if payload['source']['provider'] == 'osfstorage' \ + and source_node.type != 'osf.quickfilesnode': + source_storage_type = get_project_storage_type(source_node) + update_quota(source_node, file_size, source_storage_type, add=False) + +def update_quota(target, file_size, storage_type, add=True): + try: + if check_select_for_update(): + user_quota = UserQuota.objects.filter( + user=target.creator, + storage_type=storage_type + ).select_for_update().get() + else: + user_quota = UserQuota.objects.get( + user=target.creator, + storage_type=storage_type + ) + if add: + user_quota.used += file_size + else: + user_quota.used -= file_size + user_quota.save() + except UserQuota.DoesNotExist: + try: + with transaction.atomic(): + UserQuota.objects.create( + user=target.creator, + storage_type=storage_type, + max_quota=api_settings.DEFAULT_MAX_QUOTA, + used=file_size + ) + except IntegrityError: + if check_select_for_update(): + user_quota = UserQuota.objects.filter( + user=target.creator, + storage_type=storage_type + ).select_for_update().get() + else: + user_quota = UserQuota.objects.get( + user=target.creator, + storage_type=storage_type + ) + if add: + user_quota.used += file_size + else: + user_quota.used -= file_size + user_quota.save() + +def get_file_size(children): + children = children.get('children', []) + size = 0 + for child in children: + if child['kind'] == 'file': + size += int(child['size']) + else: + size += get_file_size(child) + return size From fa41a4ba51928d287c4c06b3905c83d791980536 Mon Sep 17 00:00:00 2001 From: hcphat Date: Wed, 29 Apr 2026 15:08:23 +0700 Subject: [PATCH 02/41] =?UTF-8?q?ref=202.2.=E3=82=A2=E3=83=89=E3=82=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AB?= =?UTF-8?q?=E7=B5=A1=E3=82=80=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E6=99=82=E3=81=AE=E3=82=AF=E3=82=AA=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=AE=A1=E7=90=86=20:=20Fix=20review=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_views_copy_move_quota.py | 217 ++++++++++++++++++ tests/test_quota.py | 29 ++- tests/test_quota_move_copy.py | 10 +- website/static/js/fangorn.js | 24 +- website/util/quota.py | 107 +++------ 5 files changed, 289 insertions(+), 98 deletions(-) diff --git a/addons/osfstorage/tests/test_views_copy_move_quota.py b/addons/osfstorage/tests/test_views_copy_move_quota.py index cc166dc5666..991b71167d0 100644 --- a/addons/osfstorage/tests/test_views_copy_move_quota.py +++ b/addons/osfstorage/tests/test_views_copy_move_quota.py @@ -8,9 +8,11 @@ - osfstorage_move_hook (file + folder, intra-target replaced_size, inter-target quota) - osfstorage_create_child (FileInfo UPSERT + quota delta on upload) - osfstorage_delete (folder quota subtraction before deletion) + - TestUploadIntegration (full upload flow: create_child -> create_waterbutler_log) """ from __future__ import unicode_literals +import time as time_module import mock import pytest from nose.tools import assert_equal, assert_true, assert_false @@ -863,3 +865,218 @@ def test_delete_file_does_not_trigger_folder_quota_path(self, mock_st, mock_uq): # (The existing file-quota path in views.py does NOT call update_quota # on delete; that is handled by the WaterButler hook elsewhere.) assert not mock_uq.called + + +# --------------------------------------------------------------------------- +# Integration: full upload flow – osfstorage_create_child + create_waterbutler_log +# --------------------------------------------------------------------------- + +@pytest.mark.django_db +class TestUploadIntegration(StorageTestCase): + """ + Integration tests simulating the complete OSF upload flow. + + Step 1 – osfstorage_create_child: + WaterButler calls this endpoint after writing the file to storage. + Creates a FileInfo record and calls update_quota ONCE. + + Step 2 – create_waterbutler_log: + WaterButler calls this endpoint to record the log. + Fires file_signals.file_updated → update_used_quota. + The skip logic (provider == 'osfstorage') prevents update_quota + from being called a SECOND time (Critical 1 fix). + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _upload(self, parent, name, size=123, unique_hash=None): + """POST to osfstorage_create_child (WaterButler upload callback).""" + payload = make_payload(user=self.user, name=name) + payload['metadata']['size'] = size + if unique_hash is not None: + # Different hash → genuinely new FileVersion (not treated as duplicate) + payload['metadata']['name'] = unique_hash + return self.app.post_json( + api_url_for( + 'osfstorage_create_child', + fid=parent._id, + guid=self.project._id, + ), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + def _wb_log(self, file_id, name, size, action='create'): + """PUT to create_waterbutler_log (WaterButler log callback).""" + options = { + 'auth': {'id': self.user._id}, + 'action': action, + 'provider': 'osfstorage', + 'metadata': { + 'provider': 'osfstorage', + 'name': name, + 'materialized': '/' + file_id, + 'path': '/' + file_id, + 'kind': 'file', + 'size': size, + 'created_utc': '', + 'modified_utc': '', + 'extra': {'version': '1'}, + }, + 'time': time_module.time() + 1000, + } + message, signature = signing.default_signer.sign_payload(options) + return self.app.put_json( + self.project.api_url_for('create_waterbutler_log'), + {'payload': message, 'signature': signature}, + headers={'Content-Type': 'application/json'}, + expect_errors=True, + ) + + # ------------------------------------------------------------------ + # Upload new file + # ------------------------------------------------------------------ + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.base.views.BaseFileNode') + @mock.patch('addons.base.views.timestamp') + def test_new_upload_wb_log_returns_200_and_no_duplicate( + self, mock_ts, mock_bfn, mock_st, mock_uq): + """ + Scenario #1: Upload a brand-new file. + + Expected: + - osfstorage_create_child: returns 200/201, creates FileInfo, calls update_quota once. + - create_waterbutler_log: returns 200 (not 500), creates a NodeLog entry. + - After both steps: still exactly 1 FileInfo record; update_quota NOT called again via signal. + """ + name = 'new_upload.txt' + size = 1024 + + # Step 1: WaterButler upload callback -> osfstorage_create_child + upload_resp = self._upload(self.root_node, name, size=size) + assert upload_resp.status_code in (200, 201), ( + f'osfstorage_create_child returned {upload_resp.status_code}' + ) + + # Scenario #4: exactly 1 FileInfo record created + record = self.root_node.find_child_by_name(name) + assert record is not None + assert FileInfo.objects.filter(file=record).count() == 1, ( + 'FileInfo must have exactly 1 record after upload.' + ) + + # Scenario #3: update_quota called exactly once from create_child + assert mock_uq.call_count == 1, ( + f'update_quota should be called once from osfstorage_create_child, ' + f'but was called {mock_uq.call_count} time(s).' + ) + mock_uq.reset_mock() + + # Step 2: WaterButler log callback -> create_waterbutler_log + nlogs = self.project.logs.count() + log_resp = self._wb_log(record._id, name, size) + + # Scenario #1: /create_waterbutler_log must return 200 + assert log_resp.status_code == 200, ( + f'create_waterbutler_log returned {log_resp.status_code} ' + '(expected 200). This was Critical 1: FILE_ADDED signal caused ' + 'duplicate FileInfo creation -> IntegrityError -> 500.' + ) + + # NodeLog must be created + self.project.reload() + assert self.project.logs.count() == nlogs + 1, ( + 'A NodeLog entry must be created by create_waterbutler_log.' + ) + + # Scenario #4: FileInfo still exactly 1 record (signal was skipped) + assert FileInfo.objects.filter(file=record).count() == 1, ( + 'FileInfo must remain exactly 1 record after create_waterbutler_log. ' + 'The skip logic in update_used_quota must prevent duplication.' + ) + + # Scenario #3: update_quota must NOT be called again via signal + assert mock_uq.call_count == 0, ( + f'update_quota was called {mock_uq.call_count} extra time(s) via ' + 'the FILE_ADDED signal. The skip logic should have prevented this.' + ) + + # ------------------------------------------------------------------ + # Overwrite: upload a new version of an existing file + # ------------------------------------------------------------------ + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.base.views.BaseFileNode') + @mock.patch('addons.base.views.timestamp') + def test_overwrite_upload_wb_log_returns_200_and_no_duplicate( + self, mock_ts, mock_bfn, mock_st, mock_uq): + """ + Scenario #2: Upload a new version of an existing file (same filename). + + Expected: + - create_waterbutler_log returns 200. + - A NodeLog entry is created. + - FileInfo is UPDATED (UPSERT), not duplicated -> still exactly 1 record. + - update_quota called once from create_child for the size delta. + """ + name = 'overwrite.txt' + size_v1, size_v2 = 500, 800 + + # First upload + self._upload(self.root_node, name, size=size_v1, unique_hash='hash_v1') + record = self.root_node.find_child_by_name(name) + assert FileInfo.objects.filter(file=record).count() == 1 + mock_uq.reset_mock() + + # Second upload (same filename, different hash -> genuinely new FileVersion) + upload_resp = self._upload( + self.root_node, name, size=size_v2, unique_hash='hash_v2' + ) + assert upload_resp.status_code in (200, 201) + record.reload() + + # FileInfo must be UPDATED, not duplicated + fi = FileInfo.objects.filter(file=record).first() + assert fi is not None + assert fi.file_size == size_v2, ( + f'FileInfo.file_size should be {size_v2} after overwrite, got {fi.file_size}.' + ) + assert FileInfo.objects.filter(file=record).count() == 1 + + # update_quota called once for the new version (delta = size_v2 - size_v1) + assert mock_uq.call_count == 1 + mock_uq.reset_mock() + + # WaterButler log callback (action='update' for an overwrite) + nlogs = self.project.logs.count() + log_resp = self._wb_log(record._id, name, size_v2, action='update') + + # Scenario #2: /create_waterbutler_log must return 200 + assert log_resp.status_code == 200, ( + f'create_waterbutler_log returned {log_resp.status_code} ' + 'for overwrite (expected 200).' + ) + + self.project.reload() + assert self.project.logs.count() == nlogs + 1 + + # Still exactly 1 FileInfo record + assert FileInfo.objects.filter(file=record).count() == 1, ( + 'FileInfo must remain exactly 1 record after overwrite + ' + 'create_waterbutler_log.' + ) + + # update_quota must NOT be called again via signal (osfstorage skip) + assert mock_uq.call_count == 0, ( + f'update_quota was called {mock_uq.call_count} extra time(s) via ' + 'signal after overwrite. Skip logic should prevent this.' + ) diff --git a/tests/test_quota.py b/tests/test_quota.py index 2fc183213d0..add3862006f 100644 --- a/tests/test_quota.py +++ b/tests/test_quota.py @@ -294,9 +294,8 @@ def test_add_file_info(self): ) file_info_list = FileInfo.objects.filter(file=self.file).all() - assert_equal(file_info_list.count(), 1) - file_info = file_info_list.first() - assert_equal(file_info.file_size, 1000) + # FileInfo was not created + assert_equal(file_info_list.count(), 0) def test_update_file_info(self): file_info = FileInfo(file=self.file, file_size=1000) @@ -414,9 +413,7 @@ def test_add_first_file(self): storage_type=UserQuota.NII_STORAGE, user=self.project_creator ).all() - assert_equal(len(user_quota), 1) - user_quota = user_quota[0] - assert_equal(user_quota.used, 1000) + assert_equal(len(user_quota), 0) def test_add_first_file_custom_storage(self): assert_false(UserQuota.objects.filter(user=self.project_creator).exists()) @@ -453,9 +450,7 @@ def test_add_first_file_custom_storage(self): storage_type=UserQuota.CUSTOM_STORAGE, user=self.project_creator ).all() - assert_equal(len(user_quota), 1) - user_quota = user_quota[0] - assert_equal(user_quota.used, 1200) + assert_equal(len(user_quota), 0) def test_add_file(self): UserQuota.objects.create( @@ -492,7 +487,7 @@ def test_add_file(self): ).all() assert_equal(len(user_quota), 1) user_quota = user_quota[0] - assert_equal(user_quota.used, 6500) + assert_equal(user_quota.used, 5500) def test_add_file_custom_storage(self): UserQuota.objects.create( @@ -536,7 +531,7 @@ def test_add_file_custom_storage(self): ).all() assert_equal(len(user_quota), 1) user_quota = user_quota[0] - assert_equal(user_quota.used, 6700) + assert_equal(user_quota.used, 5500) def test_add_file_negative_size(self): quota.update_used_quota( @@ -1358,7 +1353,7 @@ def test_delete_file_with_Amazon_S3_Compatible_Storage_for_Institution(self): if check_select_for_update(): mock_file_info.objects.filter.return_value.select_for_update.return_value.get.return_value = FileInfo( file=self.base_file_node, file_size=1000) - mock_user_quota.objects.filter.return_value.select_for_update.return_value.first.return_value = UserQuota( + mock_user_quota.objects.filter.return_value.select_for_update.return_value.get.return_value = UserQuota( user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE, max_quota=api_settings.DEFAULT_MAX_QUOTA, @@ -1366,12 +1361,14 @@ def test_delete_file_with_Amazon_S3_Compatible_Storage_for_Institution(self): ) else: mock_file_info.objects.get.return_value = FileInfo(file=self.base_file_node, file_size=1000) - mock_user_quota.objects.filter.return_value.first.return_value = UserQuota( + _real_user_quota = UserQuota( user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE, max_quota=api_settings.DEFAULT_MAX_QUOTA, used=5500 ) + mock_user_quota.objects.get.return_value = _real_user_quota + mock_user_quota.objects.filter.return_value.first.return_value = _real_user_quota with mock.patch('website.util.quota.BaseFileNode', mock_base_file_node): with mock.patch('website.util.quota.FileInfo', mock_file_info): with mock.patch('website.util.quota.UserQuota', mock_user_quota): @@ -1414,7 +1411,7 @@ def test_delete_folder_with_Amazon_S3_Compatible_Storage_for_Institution(self): if check_select_for_update(): mock_file_info.objects.filter.return_value.select_for_update.return_value.get.return_value = FileInfo( file=self.base_file_node, file_size=1500) - mock_user_quota.objects.filter.return_value.select_for_update.return_value.first.return_value = UserQuota( + mock_user_quota.objects.filter.return_value.select_for_update.return_value.get.return_value = UserQuota( user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE, max_quota=api_settings.DEFAULT_MAX_QUOTA, @@ -1422,12 +1419,14 @@ def test_delete_folder_with_Amazon_S3_Compatible_Storage_for_Institution(self): ) else: mock_file_info.objects.get.return_value = FileInfo(file=self.base_file_node, file_size=1500) - mock_user_quota.objects.filter.return_value.first.return_value = UserQuota( + _real_user_quota = UserQuota( user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE, max_quota=api_settings.DEFAULT_MAX_QUOTA, used=5500 ) + mock_user_quota.objects.get.return_value = _real_user_quota + mock_user_quota.objects.filter.return_value.first.return_value = _real_user_quota with mock.patch('website.util.quota.BaseFileNode', mock_base_file_node): with mock.patch('website.util.quota.FileInfo', mock_file_info): with mock.patch('website.util.quota.UserQuota', mock_user_quota): diff --git a/tests/test_quota_move_copy.py b/tests/test_quota_move_copy.py index c39830d94e0..21ccd7b5b0e 100644 --- a/tests/test_quota_move_copy.py +++ b/tests/test_quota_move_copy.py @@ -211,9 +211,9 @@ def test_update_quota_integrityerror_select_for_update_subtract( uq ] quota.update_quota(mock_node, 3, 1, add=False) - assert mock_filter.return_value.select_for_update.return_value.get.call_count == 2 - assert uq.used == 7 - uq.save.assert_called_once() + assert mock_filter.return_value.select_for_update.return_value.get.call_count == 1 + assert uq.used == 10 + uq.save.assert_not_called() @mock.patch('website.util.quota.check_select_for_update', return_value=False) @mock.patch('website.util.quota.transaction.atomic') @@ -244,5 +244,5 @@ def test_update_quota_integrityerror_get_subtract( uq.used = 30 mock_get.side_effect = [quota.UserQuota.DoesNotExist, uq] quota.update_quota(mock_node, 12, 1, add=False) - assert uq.used == 18 - uq.save.assert_called_once() + assert uq.used == 30 + uq.save.assert_not_called() diff --git a/website/static/js/fangorn.js b/website/static/js/fangorn.js index 0cb301ea584..461ceccbfa7 100644 --- a/website/static/js/fangorn.js +++ b/website/static/js/fangorn.js @@ -89,6 +89,10 @@ var OPERATIONS = { } }; +var MESSAGE_MAP = { + 'quota_exceeded': gettext('You do not have enough available quota.'), +}; + var isInUploadFolderProcess = false; var isOngoingUploadFolder = false; @@ -820,15 +824,17 @@ function doItemOp(operation, to, from, rename, conflict) { } if (xhr.status === 413 && xhr.responseJSON && xhr.responseJSON.oversized_files) { + var serverMaxSize = xhr.responseJSON.max_size; var destMaxSize = to.data && to.data.accept && to.data.accept.maxSize; - var maxSizeDisplay = destMaxSize ? $osf.humanFileSize(destMaxSize * 1000000, true) : '5 GB'; - + var maxSizeDisplay = serverMaxSize ? + $osf.humanFileSize(serverMaxSize, true) : + (destMaxSize ? $osf.humanFileSize(destMaxSize * 1000000, true) : null); xhr.responseJSON.oversized_files.forEach(function(fileObj) { var displaySize = $osf.humanFileSize(fileObj.size, true); - $osf.growl(sprintf( - gettext('File「%1$s」is too large (%2$s). Max file size is %3$s.'), - fileObj.name, displaySize, maxSizeDisplay - )); + var msg = maxSizeDisplay ? + sprintf(gettext('File「%1$s」is too large (%2$s). Max file size is %3$s.'), fileObj.name, displaySize, maxSizeDisplay) : + sprintf(gettext('File「%1$s」is too large (%2$s).'), fileObj.name, displaySize); + $osf.growl(msg); }); if (notRenameOp) { @@ -840,9 +846,9 @@ function doItemOp(operation, to, from, rename, conflict) { return; } if (xhr.status === 406 && xhr.responseJSON && xhr.responseJSON.message) { - $osf.growl(sprintf( - gettext(xhr.responseJSON.message) - )); + var key = xhr.responseJSON.message_key; + var msg = (key && MESSAGE_MAP[key]) ? MESSAGE_MAP[key] : xhr.responseJSON.message; + $osf.growl(msg); if (notRenameOp) { addFileStatus(tb, from, false, '', '', conflict); } diff --git a/website/util/quota.py b/website/util/quota.py index ed22033b718..31688511674 100644 --- a/website/util/quota.py +++ b/website/util/quota.py @@ -191,6 +191,9 @@ def update_used_quota(self, target, user, event_type, payload): storage_type = get_project_storage_type(target) if event_type == FileLog.FILE_ADDED: + metadata_provider = (payload.get('metadata') or {}).get('provider') + if metadata_provider == 'osfstorage': + return file_added(target, payload, file_node, storage_type) elif event_type == FileLog.FILE_REMOVED: if metadata_provider in PROVIDERS: @@ -234,76 +237,31 @@ def file_added(target, payload, file_node, storage_type): file_size = int(payload['metadata']['size']) if file_size < 0: return - try: - if check_select_for_update(): - user_quota = UserQuota.objects.filter( - user=target.creator, - storage_type=storage_type - ).select_for_update().get() - else: - user_quota = UserQuota.objects.get( - user=target.creator, - storage_type=storage_type - ) - user_quota.used += file_size - user_quota.save() - except UserQuota.DoesNotExist: + update_quota(target, file_size, storage_type, add=True) + FileInfo.objects.create(file=file_node, file_size=file_size) + +def node_removed(target, user, payload, file_node, storage_type): + if 'osf.trashed' not in file_node.type: + logging.error('FileNode is not trashed, cannot update used quota!') + return + + total_removed = 0 + for removed_file in get_node_file_list(file_node): try: - with transaction.atomic(): - UserQuota.objects.create( - user=target.creator, - storage_type=storage_type, - max_quota=api_settings.DEFAULT_MAX_QUOTA, - used=file_size - ) - except IntegrityError: if check_select_for_update(): - user_quota = UserQuota.objects.filter( - user=target.creator, - storage_type=storage_type - ).select_for_update().get() + file_info = FileInfo.objects.filter(file=removed_file).select_for_update().get() else: - user_quota = UserQuota.objects.get( - user=target.creator, - storage_type=storage_type - ) - user_quota.used += file_size - user_quota.save() + file_info = FileInfo.objects.get(file=removed_file) + except FileInfo.DoesNotExist: + logging.error('FileInfo not found, cannot update used quota!') + continue - FileInfo.objects.create(file=file_node, file_size=file_size) + total_removed += file_info.file_size + file_info.file_size = 0 + file_info.save() -def node_removed(target, user, payload, file_node, storage_type): - if check_select_for_update(): - user_quota = UserQuota.objects.filter( - user=target.creator, - storage_type=storage_type - ).select_for_update().first() - else: - user_quota = UserQuota.objects.filter( - user=target.creator, - storage_type=storage_type - ).first() - if user_quota is not None: - if 'osf.trashed' not in file_node.type: - logging.error('FileNode is not trashed, cannot update used quota!') - return - - for removed_file in get_node_file_list(file_node): - try: - if check_select_for_update(): - file_info = FileInfo.objects.filter(file=removed_file).select_for_update().get() - else: - file_info = FileInfo.objects.get(file=removed_file) - except FileInfo.DoesNotExist: - logging.error('FileInfo not found, cannot update used quota!') - continue - - user_quota.used -= file_info.file_size - if user_quota.used < 0: - user_quota.used = 0 - file_info.file_size = 0 - file_info.save() - user_quota.save() + if total_removed > 0: + update_quota(target, total_removed, storage_type, add=False) def file_modified(target, user, payload, file_node, storage_type): file_size = int(payload['metadata']['size']) @@ -452,6 +410,16 @@ def file_moved(target, payload): update_quota(source_node, file_size, source_storage_type, add=False) def update_quota(target, file_size, storage_type, add=True): + """Add or subtract file_size from UserQuota.used for target.creator. + + This is the single source of truth for UserQuota updates. + Both ``file_added`` and ``node_removed`` delegate to this function. + + :param target: AbstractNode whose creator's quota is updated + :param int file_size: size in bytes to add or subtract + :param int storage_type: UserQuota.NII_STORAGE or CUSTOM_STORAGE + :param bool add: True to add (upload), False to subtract (delete) + """ try: if check_select_for_update(): user_quota = UserQuota.objects.filter( @@ -467,8 +435,12 @@ def update_quota(target, file_size, storage_type, add=True): user_quota.used += file_size else: user_quota.used -= file_size + if user_quota.used < 0: + user_quota.used = 0 user_quota.save() except UserQuota.DoesNotExist: + if not add: + return try: with transaction.atomic(): UserQuota.objects.create( @@ -488,10 +460,7 @@ def update_quota(target, file_size, storage_type, add=True): user=target.creator, storage_type=storage_type ) - if add: - user_quota.used += file_size - else: - user_quota.used -= file_size + user_quota.used += file_size user_quota.save() def get_file_size(children): From e13fdd9f5c97d5a08f7ce6561fc0442002088806 Mon Sep 17 00:00:00 2001 From: hcphat Date: Fri, 8 May 2026 13:45:28 +0700 Subject: [PATCH 03/41] =?UTF-8?q?ref=202.2.=E3=82=A2=E3=83=89=E3=82=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AB?= =?UTF-8?q?=E7=B5=A1=E3=82=80=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E6=99=82=E3=81=AE=E3=82=AF=E3=82=AA=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=AE=A1=E7=90=86=20:=20Fix=20review=20UT=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_views_copy_move_quota.py | 41 +++++++++++++++++++ tests/test_quota.py | 6 +-- tests/test_quota_move_copy.py | 12 +++--- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/addons/osfstorage/tests/test_views_copy_move_quota.py b/addons/osfstorage/tests/test_views_copy_move_quota.py index 991b71167d0..9e1c7a36e2d 100644 --- a/addons/osfstorage/tests/test_views_copy_move_quota.py +++ b/addons/osfstorage/tests/test_views_copy_move_quota.py @@ -1080,3 +1080,44 @@ def test_overwrite_upload_wb_log_returns_200_and_no_duplicate( f'update_quota was called {mock_uq.call_count} extra time(s) via ' 'signal after overwrite. Skip logic should prevent this.' ) + + @mock.patch('addons.base.views.BaseFileNode') + @mock.patch('addons.base.views.timestamp') + def test_upload_actual_quota_increment_no_double_count(self, mock_ts, mock_bfn): + """ + Do not mock update_quota — verify UserQuota.used in the database: + - After osfstorage_create_child: used increases exactly by file_size. + - After create_waterbutler_log: used does not increase further (no double-count). + """ + from osf.models import UserQuota + + # Ensure UserQuota initially exists with used=0 + uq, _ = UserQuota.objects.get_or_create( + user=self.user, + storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 200, 'used': 0}, + ) + initial_used = UserQuota.objects.get( + user=self.user, storage_type=UserQuota.NII_STORAGE + ).used + + # Step 1: osfstorage_create_child + self._upload(self.root_node, 'real.txt', size=500) + record = self.root_node.find_child_by_name('real.txt') + assert record is not None + + after_create_child = UserQuota.objects.get( + user=self.user, storage_type=UserQuota.NII_STORAGE + ).used + assert after_create_child - initial_used == 500, \ + f'Initial increment incorrect: expected +500, got +{after_create_child - initial_used}' + + # Step 2: create_waterbutler_log + log_resp = self._wb_log(record._id, 'real.txt', 500) + assert log_resp.status_code == 200 + + final = UserQuota.objects.get( + user=self.user, storage_type=UserQuota.NII_STORAGE + ).used + assert final == after_create_child, \ + f'No double-counting after create_waterbutler_log: expected {after_create_child}, got {final}' diff --git a/tests/test_quota.py b/tests/test_quota.py index add3862006f..a6d73106cf7 100644 --- a/tests/test_quota.py +++ b/tests/test_quota.py @@ -268,7 +268,7 @@ def setUp(self): ) self.file.save() - def test_add_file_info(self): + def test_file_info_skipped_for_osfstorage_signal(self): file_info_query = FileInfo.objects.filter(file=self.file) assert_false(file_info_query.exists()) @@ -385,7 +385,7 @@ def setUp(self): target_content_type_id=2 ) - def test_add_first_file(self): + def test_first_file_userquota_not_created_for_osfstorage_signal(self): assert_false(UserQuota.objects.filter(user=self.project_creator).exists()) quota.update_used_quota( @@ -452,7 +452,7 @@ def test_add_first_file_custom_storage(self): ).all() assert_equal(len(user_quota), 0) - def test_add_file(self): + def test_userquota_unchanged_for_osfstorage_signal(self): UserQuota.objects.create( user=self.project_creator, storage_type=UserQuota.NII_STORAGE, diff --git a/tests/test_quota_move_copy.py b/tests/test_quota_move_copy.py index 21ccd7b5b0e..114113ebbf3 100644 --- a/tests/test_quota_move_copy.py +++ b/tests/test_quota_move_copy.py @@ -35,18 +35,18 @@ def test_get_file_size_nested(): @mock.patch('website.util.quota.ProjectStorageType.objects.get') @mock.patch('website.util.quota.AbstractNode.objects.get') @mock.patch('website.util.quota.update_quota') -def test_handle_move_copy_dest_osfstorage(mock_update_quota, mock_node_get, mock_pst_get, mock_node): - # Mock ProjectStorageType.objects.get to return a mock with storage_type +@pytest.mark.django_db +def test_handle_move_copy_source_extend_dest_osfstorage(mock_update_quota, mock_node_get, mock_pst_get, mock_node): + """source=s3compatinstitutions, dest=osfstorage → update_quota không được gọi vì source không phải osfstorage""" mock_pst_get.return_value = mock.Mock(storage_type=1) - # Mock AbstractNode.objects.get to return a mock node mock_node_get.return_value = mock_node payload = { - 'destination': {'provider': 's3compatinstitutions', 'size': 123, 'children': None}, - 'source': {'nid': 'nid1', 'provider': 'osfstorage'} + 'destination': {'provider': 'osfstorage', 'size': 123, 'children': None}, + 'source': {'nid': 'nid1', 'provider': 's3compatinstitutions'} } quota._handle_move_copy(quota.FileLog.FILE_MOVED, mock_node, mock.Mock(), payload) - mock_update_quota.assert_called_once() + mock_update_quota.assert_not_called() @mock.patch('website.util.quota.ProjectStorageType.objects.get') @mock.patch('website.util.quota.AbstractNode.objects.get') From 16e472ae280503d99fc3e7aa036afed04c2cb8ae Mon Sep 17 00:00:00 2001 From: hcphat Date: Tue, 2 Jun 2026 15:39:28 +0700 Subject: [PATCH 04/41] =?UTF-8?q?ref=202.2.=E3=82=A2=E3=83=89=E3=82=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AB?= =?UTF-8?q?=E7=B5=A1=E3=82=80=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E6=99=82=E3=81=AE=E3=82=AF=E3=82=AA=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=AE=A1=E7=90=86=20:=20Rework=20review=20comment=20=20+=20Upd?= =?UTF-8?q?ate=20email=20template=20handle=20move/copy=20file=20quota=20an?= =?UTF-8?q?d=20max=20size=20error=20=20+=20Add=20script=20migrate=20missin?= =?UTF-8?q?g=20file=20info=20and=20update=20quota=20used?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/base/views.py | 32 +- .../migration/migrate_fix_fileinfo_quota.py | 533 ++++++++++++++++++ tests/test_addons.py | 95 +++- .../emails/file_operation_failed.html.mako | 26 +- 4 files changed, 682 insertions(+), 4 deletions(-) create mode 100644 scripts/migration/migrate_fix_fileinfo_quota.py diff --git a/addons/base/views.py b/addons/base/views.py index e6f75bbb5e7..eb63448a331 100644 --- a/addons/base/views.py +++ b/addons/base/views.py @@ -66,6 +66,8 @@ from website.ember_osf_web.decorators import ember_flag_is_active from website.project.utils import serialize_node from website.util import rubeus, timestamp, inspect_info # noqa +import re +import json as _json from osf.features import ( SLOAN_COI_DISPLAY, @@ -585,6 +587,33 @@ def create_waterbutler_log(payload, **kwargs): params=payload ) if payload.get('email') is True or payload.get('errors'): + # Parse structured error info from WaterButler exception repr strings + # e.g. "" + error_info = None + if payload.get('errors'): + for err_str in payload['errors']: + # Try to extract JSON payload embedded in the repr string + json_match = re.search(r'\((?:\d+),\s*(\{.*\})\)', err_str, re.DOTALL) + if json_match: + try: + err_data = _json.loads(json_match.group(1)) + message_key = err_data.get('message_key', '') + if message_key == 'quota_exceeded' or 'quota_exceeded' in err_str: + error_info = {'type': 'quota_exceeded'} + break + elif err_data.get('oversized_files'): + error_info = { + 'type': 'oversized', + 'oversized_files': err_data['oversized_files'], + 'max_size': err_data.get('max_size'), + } + break + except Exception as e: + logger.warning( + f'Failed to parse WaterButler error representation string: {err_str}. ' + f'Error: {e}' + ) + mails.send_mail( user.username, mails.FILE_OPERATION_FAILED if payload.get('errors') @@ -595,7 +624,8 @@ def create_waterbutler_log(payload, **kwargs): source_path=payload['source']['materialized'], source_addon=payload['source']['addon'], destination_addon=payload['destination']['addon'], - osf_support_email=settings.OSF_SUPPORT_EMAIL + osf_support_email=settings.OSF_SUPPORT_EMAIL, + error_info=error_info, ) if payload.get('errors'): # Action failed but our function succeeded diff --git a/scripts/migration/migrate_fix_fileinfo_quota.py b/scripts/migration/migrate_fix_fileinfo_quota.py new file mode 100644 index 00000000000..e2fd73ba38a --- /dev/null +++ b/scripts/migration/migrate_fix_fileinfo_quota.py @@ -0,0 +1,533 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +migrate_fix_fileinfo_quota.py + +Fix missing or zero-size FileInfo records and recalculate UserQuota. + +Background +---------- +During Move/Copy operations, if a system error occurs after the file has been +written but before the FileInfo hook completes, the resulting OsfStorageFile may +have no FileInfo row (or a row with file_size=0). This causes UserQuota.used to +be under-counted for the affected project creator. + +This script: +1. Finds all non-deleted OsfStorageFile nodes that are missing a FileInfo record + OR have a FileInfo record with file_size=0. +2. Populates file_size from the latest FileVersion.size (which is authoritative). +3. Collects all unique project creators affected by the above. +4. Calls update_user_used_quota(user) for each affected user to re-sync + UserQuota.used from the actual FileInfo totals. + Storage type is determined per-project (NII_STORAGE or CUSTOM_STORAGE). +5. Exports a CSV report of all processed records, including skipped files. + +Database Transactions: + All database updates (FileInfo insertions/updates and UserQuota recalculations) + are wrapped in a single database transaction (transaction.atomic). If the script + crashes or encounters an unhandled exception during the fix process, all changes + are rolled back to ensure database consistency. + +Usage: + # Fix and export results to CSV + python -m scripts.migration.migrate_fix_fileinfo_quota + python -m scripts.migration.migrate_fix_fileinfo_quota --output result.csv + + # Preview changes and export to CSV without writing to DB (dry run) + python -m scripts.migration.migrate_fix_fileinfo_quota --dry + python -m scripts.migration.migrate_fix_fileinfo_quota --dry --output preview.csv +""" + +import sys +import os +import csv +import logging +import argparse +from datetime import datetime + +import django + +# Setup Django before importing models +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'api.base.settings') +django.setup() + +from django.db import transaction +from django.db.models import Q + +from addons.osfstorage.models import OsfStorageFile +from osf.models import FileInfo, OSFUser, UserQuota, ProjectStorageType +from website.util.quota import update_user_used_quota, get_project_storage_type +from scripts import utils as script_utils +from django.contrib.contenttypes.models import ContentType +from osf.models import Guid + +logger = logging.getLogger(__name__) + +RESULT_FIELDNAMES = [ + # osf_file_id : _id (hex string) of the OsfStorageFile / BaseFileNode record + # fileinfo_id : integer PK of the FileInfo record inserted/updated (empty in dry_run) + # fileinfo__id : _id (hex string) of the FileInfo record inserted/updated (empty in dry_run) + # fileinfo_file_id: integer PK of BaseFileNode — the value stored as file_id (FK) in the FileInfo table + # fileinfo_created: True = new FileInfo row INSERT-ed, False = existing row UPDATE-d (empty in dry_run) + # action : would_update | created | updated | updated_no_quota | skipped + # quota_recalculated: True if quota was recalculated for this file's creator + 'osf_file_id', 'file_name', + 'fileinfo_id', 'fileinfo__id', 'fileinfo_file_id', + 'file_size_bytes', 'project_guid', 'creator_id', 'storage_type', + 'fileinfo_created', 'action', 'quota_recalculated', 'dry_run', +] + + +# --- Scanning Operations --- + +def get_broken_file_nodes(): + """Return queryset of OsfStorageFile nodes that: + - are not deleted (deleted_on is None) + - either have no FileInfo, or have FileInfo with file_size == 0 + + Optimized using a LEFT OUTER JOIN query to avoid subquery bottlenecks with large datasets. + """ + return ( + OsfStorageFile.objects + .filter(deleted_on=None, deleted_by_id=None) + .filter(Q(fileinfo__isnull=True) | Q(fileinfo__file_size=0)) + # NOTE: Do NOT use prefetch_related('target') here. + # BaseFileNode uses TypedModel + GenericForeignKey; Django's GFK prefetch cache + # is incompatible with TypedModel proxy resolution and returns None even when + # target_content_type_id is valid. Use direct lookup instead (see resolve_target). + ) + + +def _bulk_get_latest_version_sizes(osf_file_ids): + """Fetch the latest FileVersion size for each OsfStorageFile in bulk. + Returns a dict mapping osf_file.id -> size (int or None). + """ + if not osf_file_ids: + return {} + + # Access the M2M through-model (BaseFileVersionsThrough) without importing it directly. + # Fields: basefilenode_id (FK → BaseFileNode), fileversion_id (FK → FileVersion) + ThroughModel = OsfStorageFile.versions.through + + # Single JOIN query: fetches all through-entries + related FileVersion data at once. + entries = ( + ThroughModel.objects + .filter(basefilenode_id__in=osf_file_ids) + .select_related('fileversion') + ) + + # Group by file and pick the version with the highest numeric identifier. + # result maps: file_node_id → (version_number_int, size) + best = {} # file_node_id -> (ver_num, size) + for entry in entries: + fv = entry.fileversion + file_node_id = entry.basefilenode_id + + # identifier is a text field ('1', '2', ...) — parse to int for correct ordering + try: + ver_num = int(fv.identifier) + except (TypeError, ValueError): + ver_num = -1 # non-numeric identifiers rank last + + if file_node_id not in best or ver_num > best[file_node_id][0]: + # Resolve size: prefer fv.size, fall back to metadata dict + size = fv.size + if size is None and isinstance(fv.metadata, dict): + try: + size = int(fv.metadata.get('size')) + except (TypeError, ValueError): + size = None + best[file_node_id] = (ver_num, size) + + return {file_id: info[1] for file_id, info in best.items()} + + +def scan_broken_files(): + """Scan for OsfStorageFile nodes with missing or zero-size FileInfo. + + Returns a list of dicts with scan results (no DB writes). + Skipped files are also included with action='skipped' for full audit visibility. + """ + broken_files = list(get_broken_file_nodes()) + total = len(broken_files) + logger.info('Scanning OsfStorageFile records...') + logger.info('Found {} file node(s) with missing or zero-size FileInfo.'.format(total)) + + # Bulk-fetch latest version sizes to avoid N+1 queries + file_ids = [f.id for f in broken_files] + size_map = _bulk_get_latest_version_sizes(file_ids) + + rows = [] + for idx, osf_file in enumerate(broken_files, 1): + size = size_map.get(osf_file.id) + + if size is None or size <= 0: + logger.warning( + '[{}/{}] Skipping {} (_id={}) — could not determine size from FileVersion.'.format( + idx, total, osf_file.name, osf_file._id + ) + ) + # Include skipped files in CSV for full audit trail + rows.append({ + 'osf_file_id': osf_file._id, + 'fileinfo_file_id': osf_file.id, + 'file_name': osf_file.name, + 'file_size_bytes': size if size is not None else '', + 'project_guid': '', + 'creator_id': '', + 'storage_type': None, + 'action': 'skipped', + 'quota_recalculated': False, + 'osf_file': osf_file, + }) + continue + + project_guid = '' + creator_id = '' + storage_type_value = None + try: + # Direct lookup via ContentType to avoid GFK+TypedModel prefetch incompatibility. + # osf_file.target (GFK accessor) returns None when used after prefetch_related + # because TypedModel proxy resolution breaks the GFK prefetch cache. + target = None + if osf_file.target_content_type_id and osf_file.target_object_id: + ct = ContentType.objects.get_for_id(osf_file.target_content_type_id) + model_class = ct.model_class() + if model_class: + target = model_class.objects.filter(id=osf_file.target_object_id).first() + + if target is not None: + first_guid = target.guids.values_list('_id', flat=True).first() + project_guid = first_guid or '' + creator = getattr(target, 'creator', None) + if creator: + creator_id = creator._id + # Determine storage type for this project + storage_type_value = get_project_storage_type(target) + else: + # Target object not found — try Guid table as last resort + if osf_file.target_content_type_id and osf_file.target_object_id: + ct = ContentType.objects.get_for_id(osf_file.target_content_type_id) + guid_obj = Guid.objects.filter( + content_type=ct, + object_id=osf_file.target_object_id + ).first() + if guid_obj: + project_guid = guid_obj._id + logger.warning( + 'File {} target not found (target_content_type_id={}, target_object_id={}). ' + 'Resolved project_guid={}'.format( + osf_file._id, osf_file.target_content_type_id, osf_file.target_object_id, project_guid + ) + ) + except Exception as e: + logger.warning('Could not resolve creator/guid for file {}: {}'.format(osf_file._id, e)) + + rows.append({ + 'osf_file_id': osf_file._id, # hex _id string of OsfStorageFile + 'fileinfo_file_id': osf_file.id, # integer PK stored as file_id FK in FileInfo table + 'file_name': osf_file.name, + 'file_size_bytes': size, + 'project_guid': project_guid, + 'creator_id': creator_id, + 'storage_type': storage_type_value, + 'osf_file': osf_file, + }) + + eligible = sum(1 for r in rows if r.get('action') != 'skipped') + logger.info('Scan complete. {} record(s) eligible for fix, {} skipped.'.format( + eligible, total - eligible + )) + return rows + + +def print_results(rows): + """Display scanned records in a formatted summary table.""" + eligible = [r for r in rows if r.get('action') != 'skipped'] + skipped = [r for r in rows if r.get('action') == 'skipped'] + + if not eligible and not skipped: + print(' No broken FileInfo records found.') + return + + if eligible: + print('\n' + '-' * 160) + print(' {:<26} {:<16} {:<22} {:<18} {:<16} {:<14} {:<18}'.format( + 'OSF_FILE_ID (_id)', 'FILEINFO_FILE_ID', 'FILE_NAME', 'PROJECT_GUID', 'CREATOR_ID', 'STORAGE_TYPE', 'SIZE (bytes)' + )) + print('-' * 160) + for r in eligible: + print(' {:<26} {:<16} {:<22} {:<18} {:<16} {:<14} {:<18}'.format( + r['osf_file_id'], + r['fileinfo_file_id'], + (r['file_name'] or '')[:20], + r['project_guid'], + r['creator_id'], + str(r.get('storage_type') if r.get('storage_type') is not None else ''), + r['file_size_bytes'], + )) + print('-' * 160) + print(' Total: {} record(s) would be updated.\n'.format(len(eligible))) + + if skipped: + print(' Skipped {} record(s) (no valid FileVersion size):'.format(len(skipped))) + for r in skipped: + print(' - {} ({})'.format(r['osf_file_id'], r['file_name'])) + print() + + +# --- Updating Operations --- + +def apply_fixes(rows, dry_run=False): + """Apply FileInfo fixes and recalculate UserQuota for affected users. + + This function wraps all database modifications in a single database transaction + when dry_run is False. If any unhandled exception occurs, the transaction is + rolled back, ensuring database consistency. + + If dry_run is True, no database writes are performed. + Returns the list of result rows with 'action', 'quota_recalculated', and 'dry_run' fields added. + """ + if dry_run: + logger.info('[DRY RUN] Simulating FileInfo updates (no DB writes)...') + return _apply_fixes_internal(rows, dry_run=True) + else: + logger.info('Applying FileInfo updates (wrapped in database transaction)...') + with transaction.atomic(): + return _apply_fixes_internal(rows, dry_run=False) + + +def _apply_fixes_internal(rows, dry_run=False): + """Internal helper to apply fixes, called inside/outside transaction.""" + result_rows = [] + affected_user_storage = set() # set of (creator_id, storage_type) + fixed = 0 + skipped = 0 + + for row in rows: + if row.get('action') == 'skipped': + result_rows.append({ + 'osf_file_id': row['osf_file_id'], + 'fileinfo_file_id': row['fileinfo_file_id'], + 'file_name': row['file_name'], + 'file_size_bytes': row['file_size_bytes'], + 'project_guid': row['project_guid'], + 'creator_id': row['creator_id'], + 'storage_type': row['storage_type'], + 'fileinfo_id': '', + 'fileinfo__id': '', + 'fileinfo_created': '', + 'action': 'skipped', + 'quota_recalculated': False, + 'dry_run': dry_run, + }) + skipped += 1 + continue + + # Optimize: reuse osf_file object from scan, avoiding re-querying + osf_file = row.get('osf_file') + if osf_file is None: + osf_file = OsfStorageFile.objects.filter(_id=row['osf_file_id']).first() + + if osf_file is None: + logger.warning('File {} not found, skipping.'.format(row['osf_file_id'])) + result_rows.append({ + 'osf_file_id': row['osf_file_id'], + 'fileinfo_file_id': row['fileinfo_file_id'], + 'file_name': row['file_name'], + 'file_size_bytes': row['file_size_bytes'], + 'project_guid': row['project_guid'], + 'creator_id': row['creator_id'], + 'storage_type': row['storage_type'], + 'fileinfo_id': '', + 'fileinfo__id': '', + 'fileinfo_created': '', + 'action': 'skipped', + 'quota_recalculated': False, + 'dry_run': dry_run, + }) + skipped += 1 + continue + + fileinfo_id = '' + fileinfo__id = '' + fileinfo_created = '' + if not dry_run: + fileinfo_obj, created = FileInfo.objects.update_or_create( + file=osf_file, + defaults={'file_size': row['file_size_bytes']}, + ) + fileinfo_id = fileinfo_obj.id # integer PK of FileInfo + fileinfo__id = fileinfo_obj._id # hex _id string of FileInfo + fileinfo_created = created # True = INSERT, False = UPDATE + + # Determine whether quota will actually be recalculated + creator_id = row.get('creator_id', '') + storage_type = row.get('storage_type', None) + quota_will_recalculate = bool(creator_id and storage_type is not None) + + if quota_will_recalculate: + # Track (creator_id, storage_type) pair + affected_user_storage.add((creator_id, storage_type)) + + # Use distinct action when creator_id is missing + if dry_run: + action = 'would_update' + elif not quota_will_recalculate: + # FileInfo was fixed but quota cannot be recalculated (no creator resolved) + action = 'updated_no_quota' + else: + action = 'created' if fileinfo_created else 'updated' + + logger.info('{} {} (osf_file_id={}, fileinfo_id={}, fileinfo__id={}, fileinfo_file_id={}, created={}) file_size={} bytes storage_type={}'.format( + '[DRY RUN]' if dry_run else 'Fixed:', + row['file_name'], row['osf_file_id'], + fileinfo_id, fileinfo__id, row['fileinfo_file_id'], + fileinfo_created, row['file_size_bytes'], storage_type, + )) + + if action == 'updated_no_quota': + logger.warning( + 'File {} FileInfo fixed but creator_id is empty — UserQuota NOT recalculated. ' + 'Manual review required. project_guid={}'.format( + row['osf_file_id'], row.get('project_guid', '') + ) + ) + + result_rows.append({ + 'osf_file_id': row['osf_file_id'], + 'fileinfo_file_id': row['fileinfo_file_id'], + 'file_name': row['file_name'], + 'file_size_bytes': row['file_size_bytes'], + 'project_guid': row['project_guid'], + 'creator_id': row['creator_id'], + 'storage_type': row['storage_type'], + 'fileinfo_id': fileinfo_id, + 'fileinfo__id': fileinfo__id, + 'fileinfo_created': fileinfo_created, + 'action': action, + 'quota_recalculated': False, # will be updated below after quota recalc + 'dry_run': dry_run, + }) + fixed += 1 + + logger.info('FileInfo backfill complete. fixed={}, skipped={} (dry={})'.format( + fixed, skipped, dry_run + )) + + # --- Recalculate UserQuota.used for all affected (user, storage_type) pairs --- + logger.info('Recalculating UserQuota.used for {} unique (user, storage_type) pair(s).'.format( + len(affected_user_storage) + )) + + quota_updated = 0 + quota_errors = 0 + # Track which creator_ids had successful quota recalculation + recalculated_user_storage = set() + + for user_id, storage_type in sorted(affected_user_storage): + try: + user = OSFUser.load(user_id) + if user is None: + logger.warning('User {} not found, skipping quota recalc.'.format(user_id)) + continue + + if not dry_run: + # Update quota for the actual storage_type of the project + update_user_used_quota(user, storage_type=storage_type) + + logger.info('{} Recalculated quota for user {} (storage_type={})'.format( + '[DRY RUN]' if dry_run else '', user_id, storage_type + )) + quota_updated += 1 + recalculated_user_storage.add((user_id, storage_type)) + except Exception as e: + logger.error('Failed to recalculate quota for user {} (storage_type={}): {}'.format( + user_id, storage_type, e + )) + quota_errors += 1 + + logger.info('Quota recalculation complete. updated={}, errors={} (dry={})'.format( + quota_updated, quota_errors, dry_run + )) + + # Update quota_recalculated flag in result rows + for row in result_rows: + creator_id = row.get('creator_id', '') + storage_type = row.get('storage_type', None) + if dry_run and creator_id and storage_type is not None: + # In dry-run mode, mark as "would recalculate" + row['quota_recalculated'] = 'would_recalculate' + elif (creator_id, storage_type) in recalculated_user_storage: + row['quota_recalculated'] = True + + return result_rows + + +# --- CSV Export --- + +def save_result_csv(result_rows, output_path): + """Save full results to CSV file.""" + with open(output_path, 'w', newline='', encoding='utf-8-sig') as f: + writer = csv.DictWriter(f, fieldnames=RESULT_FIELDNAMES) + writer.writeheader() + writer.writerows(result_rows) + total = len(result_rows) + skipped = sum(1 for r in result_rows if r.get('action') == 'skipped') + no_quota = sum(1 for r in result_rows if r.get('action') == 'updated_no_quota') + print('\n Exported {} result(s) to: {}'.format(total, output_path)) + if skipped: + print(' - {} skipped (no valid FileVersion size)'.format(skipped)) + if no_quota: + print(' - {} updated_no_quota (FileInfo fixed but quota NOT recalculated — manual review needed)'.format(no_quota)) + + +# --- Main --- + +def main(): + ts = datetime.now().strftime('%Y%m%d_%H%M%S') + default_output = f'migrate_fix_fileinfo_quota_{ts}.csv' + + parser = argparse.ArgumentParser( + description='Fix missing/zero-size FileInfo records and recalculate UserQuota.', + ) + parser.add_argument( + '--output', '-o', + default=default_output, + help=f'Output CSV file path (default: {default_output})', + ) + parser.add_argument( + '--dry', action='store_true', default=False, + help='Run in dry-run mode (do not commit updates to the database).', + ) + + args = parser.parse_args() + + if not args.dry: + script_utils.add_file_logger(logger, __file__) + + # Configure console logging format + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s', + ) + + # 1. Scan for broken records + rows = scan_broken_files() + + if not rows: + print('No broken FileInfo records found.') + return + + # 2. Print summary table + print_results(rows) + + # 3. Apply fixes (or simulate in dry-run mode) + result_rows = apply_fixes(rows, dry_run=args.dry) + + # 4. Export results to CSV + save_result_csv(result_rows, args.output) + + +if __name__ == '__main__': + main() diff --git a/tests/test_addons.py b/tests/test_addons.py index da39ce95634..a3a46180e93 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -25,7 +25,7 @@ DraftRegistrationFactory, ExportDataLocationFactory, ) -from website import settings +from website import settings, mails from api.base import settings as api_settings from addons.base import views from addons.github.exceptions import ApiError @@ -1023,6 +1023,99 @@ def test_add_folder_osfstorage_log(self, mock_basefilenode): assert_equal(self.node.logs.count(), nlogs + 1) assert('urls' not in self.node.logs.filter(action='osf_storage_file_added')[0].params) + @mock.patch('website.mails.send_mail') + def test_create_waterbutler_log_quota_exceeded_error(self, mock_send_mail): + url = self.node.api_url_for('create_waterbutler_log') + payload = self.build_payload( + action='move', + metadata={'path': 'pizza'}, + source={ + 'provider': 'github', + 'materialized': '/src.txt', + 'path': '/src.txt', + 'name': 'src.txt', + 'nid': self.node._id, + }, + destination={ + 'provider': 'github', + 'materialized': '/dest.txt', + 'path': '/dest.txt', + 'name': 'dest.txt', + 'nid': self.node._id, + }, + errors=[''] + ) + res = self.app.put_json(url, payload, headers={'Content-Type': 'application/json'}) + assert_equal(res.status_code, 200) + assert_true(mock_send_mail.called) + call_args = mock_send_mail.call_args[1] + assert_equal(call_args['error_info'], {'type': 'quota_exceeded'}) + assert_equal(mock_send_mail.call_args[0][1], mails.FILE_OPERATION_FAILED) + + @mock.patch('website.mails.send_mail') + def test_create_waterbutler_log_oversized_error(self, mock_send_mail): + url = self.node.api_url_for('create_waterbutler_log') + payload = self.build_payload( + action='move', + metadata={'path': 'pizza'}, + source={ + 'provider': 'github', + 'materialized': '/src.txt', + 'path': '/src.txt', + 'name': 'src.txt', + 'nid': self.node._id, + }, + destination={ + 'provider': 'github', + 'materialized': '/dest.txt', + 'path': '/dest.txt', + 'name': 'dest.txt', + 'nid': self.node._id, + }, + errors=[''] + ) + res = self.app.put_json(url, payload, headers={'Content-Type': 'application/json'}) + assert_equal(res.status_code, 200) + assert_true(mock_send_mail.called) + call_args = mock_send_mail.call_args[1] + assert_equal(call_args['error_info'], { + 'type': 'oversized', + 'oversized_files': [{'name': 'large.zip', 'size': 2000000000}], + 'max_size': 1000000000 + }) + assert_equal(mock_send_mail.call_args[0][1], mails.FILE_OPERATION_FAILED) + + @mock.patch('addons.base.views.logger.warning') + @mock.patch('website.mails.send_mail') + def test_create_waterbutler_log_unparseable_error(self, mock_send_mail, mock_logger_warning): + url = self.node.api_url_for('create_waterbutler_log') + payload = self.build_payload( + action='move', + metadata={'path': 'pizza'}, + source={ + 'provider': 'github', + 'materialized': '/src.txt', + 'path': '/src.txt', + 'name': 'src.txt', + 'nid': self.node._id, + }, + destination={ + 'provider': 'github', + 'materialized': '/dest.txt', + 'path': '/dest.txt', + 'name': 'dest.txt', + 'nid': self.node._id, + }, + errors=[''] + ) + res = self.app.put_json(url, payload, headers={'Content-Type': 'application/json'}) + assert_equal(res.status_code, 200) + assert_true(mock_send_mail.called) + assert_true(mock_logger_warning.called) + call_args = mock_send_mail.call_args[1] + assert_equal(call_args['error_info'], None) + assert_equal(mock_send_mail.call_args[0][1], mails.FILE_OPERATION_FAILED) + class TestAddonLogsDifferentProvider(OsfTestCase): def setUp(self): diff --git a/website/templates/emails/file_operation_failed.html.mako b/website/templates/emails/file_operation_failed.html.mako index 2c72fed9b16..16adf99fc6e 100644 --- a/website/templates/emails/file_operation_failed.html.mako +++ b/website/templates/emails/file_operation_failed.html.mako @@ -59,8 +59,30 @@ - An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'}. - Please log in and try this action again. If the problem persists, please email ${osf_support_email}. + <% + ei = context.get('error_info') or {} + err_type = ei.get('type', '') + %> + % if err_type == 'quota_exceeded': + An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'} because you do not have enough available storage quota. + Please free up storage space or contact your administrator to increase your quota, then try again. If the problem persists, please email ${osf_support_email}. + % elif err_type == 'oversized': + <% + oversized_files = ei.get('oversized_files', []) + max_size_bytes = ei.get('max_size') or 0 + max_size_mb = max_size_bytes // (1024 * 1024) if max_size_bytes else 0 + %> + An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'} because the following file(s) exceed the maximum allowed file size% if max_size_mb: of ${max_size_mb} MB% endif: +
      + % for f in oversized_files: +
    • ${f.get('name', 'Unknown file')} (${'{:.1f}'.format(f.get('size', 0) / (1024 * 1024))} MB)
      • + % endfor +
    + Please reduce the file size and try again. If the problem persists, please email ${osf_support_email}. + % else: + An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'}. + Please log in and try this action again. If the problem persists, please email ${osf_support_email}. + % endif From 4aaecab36ac956d7ab0501998924922bbbc7a57a Mon Sep 17 00:00:00 2001 From: hcphat Date: Wed, 10 Jun 2026 11:30:37 +0700 Subject: [PATCH 05/41] =?UTF-8?q?ref=202.2.=E3=82=A2=E3=83=89=E3=82=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AB?= =?UTF-8?q?=E7=B5=A1=E3=82=80=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E6=99=82=E3=81=AE=E3=82=AF=E3=82=AA=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=AE=A1=E7=90=86=20:=20Update=20script=20use=20is=5Frecalcula?= =?UTF-8?q?ting=5Fquota?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/migration/migrate_fix_fileinfo_quota.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/migration/migrate_fix_fileinfo_quota.py b/scripts/migration/migrate_fix_fileinfo_quota.py index e2fd73ba38a..406ceeef355 100644 --- a/scripts/migration/migrate_fix_fileinfo_quota.py +++ b/scripts/migration/migrate_fix_fileinfo_quota.py @@ -434,7 +434,7 @@ def _apply_fixes_internal(rows, dry_run=False): if not dry_run: # Update quota for the actual storage_type of the project - update_user_used_quota(user, storage_type=storage_type) + update_user_used_quota(user, storage_type=storage_type, is_recalculating_quota=True) logger.info('{} Recalculated quota for user {} (storage_type={})'.format( '[DRY RUN]' if dry_run else '', user_id, storage_type From f65c169ca4dd1675a84f61175258e487075d9ade Mon Sep 17 00:00:00 2001 From: hcphat Date: Fri, 5 Jun 2026 19:26:41 +0700 Subject: [PATCH 06/41] =?UTF-8?q?ref=202.2.=E3=82=A2=E3=83=89=E3=82=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AB?= =?UTF-8?q?=E7=B5=A1=E3=82=80=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E6=99=82=E3=81=AE=E3=82=AF=E3=82=AA=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=AE=A1=E7=90=86:=20Fix=20IT=20error=20=20+=20Conflict=20reso?= =?UTF-8?q?lve=20=20+=20Mail=20template=20render=20=20+=20Issue=20data.siz?= =?UTF-8?q?e=20is=20string=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- website/static/js/fangorn.js | 42 +++++++++++++++---- .../emails/file_operation_failed.html.mako | 2 +- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/website/static/js/fangorn.js b/website/static/js/fangorn.js index be7042faf14..394b0cf8133 100644 --- a/website/static/js/fangorn.js +++ b/website/static/js/fangorn.js @@ -606,6 +606,25 @@ function _findOversizedFiles(item, maxSizeBytes) { return oversized; } +/** + * Helper function to reject a conflict item (e.g. if file size or quota check fails) + * and advance the sync move/copy queue. + */ +function _rejectConflictItem(tb, from, to, notRenameOp, conflict) { + from.inProgress = false; + if (notRenameOp) { + addFileStatus(tb, from, false, '', '', conflict); + // Use object identity (not data.id): for MOVE, .done() replaces from.data + // in-place so from.data.id changes before .always() runs. + tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function(file) { + return file !== from; + }); + if (!tb.pendingReadyFiles.length) { + doSyncMove(tb, to.data.provider); + } + } +} + function doItemOp(operation, to, from, rename, conflict) { var tb = this; // dismiss old modal immediately to prevent button mashing @@ -663,7 +682,7 @@ function doItemOp(operation, to, from, rename, conflict) { oversized.data.name, displaySize, maxSizeDisplay )); }); - from.inProgress = false; + _rejectConflictItem(tb, from, to, notRenameOp, conflict); return; } } @@ -673,11 +692,11 @@ function doItemOp(operation, to, from, rename, conflict) { // Sum size of all files in the item being moved/copied var totalMoveSize = 0; if (from.kind === 'file') { - totalMoveSize = from.data.size || 0; + totalMoveSize = parseInt(from.data.size, 10) || 0; } else { getAllChildren(from).forEach(function(child) { if (child.kind === 'file') { - totalMoveSize += child.data.size || 0; + totalMoveSize += parseInt(child.data.size, 10) || 0; } }); } @@ -700,7 +719,7 @@ function doItemOp(operation, to, from, rename, conflict) { quotaMsgText = sprintf(gettext('Not enough quota to move/copy. The total size of the folder %1$s.'), folderSize); } from.notify.update(quotaMsgText, 'warning', undefined, 3000); - from.inProgress = false; + _rejectConflictItem(tb, from, to, notRenameOp, conflict); return; } if (quotaData.used + totalMoveSize > quotaData.max * window.contextVars.threshold) { @@ -719,6 +738,9 @@ function doItemOp(operation, to, from, rename, conflict) { } var origFrom = Object.assign({}, from); + // Must capture before COPY rebinds `from` to a new clone. + // .done() replaces from.data in-place, so ID-based filtering breaks for MOVE ops. + var originalFrom = from; if (operation === OPERATIONS.COPY) { from = tb.createItem($.extend(true, {status: operation.status}, from.data), to.id); } else { @@ -841,8 +863,8 @@ function doItemOp(operation, to, from, rename, conflict) { addFileStatus(tb, from, false, '', '', conflict); } orderFolder.call(tb, from.parent()); - tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function (file) { return file.data.id !== from.data.id; }); - from.inProgress = false; + // NOTE: pendingReadyFiles filter, from.inProgress reset, and doSyncMove + // are all handled by .always() below. Do NOT duplicate them here. return; } if (xhr.status === 406 && xhr.responseJSON && xhr.responseJSON.message) { @@ -853,8 +875,8 @@ function doItemOp(operation, to, from, rename, conflict) { addFileStatus(tb, from, false, '', '', conflict); } orderFolder.call(tb, from.parent()); - tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function (file) { return file.data.id !== from.data.id; }); - from.inProgress = false; + // NOTE: pendingReadyFiles filter, from.inProgress reset, and doSyncMove + // are all handled by .always() below. Do NOT duplicate them here. return; } @@ -882,7 +904,9 @@ function doItemOp(operation, to, from, rename, conflict) { orderFolder.call(tb, from.parent()); }).always(function(){ - tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function (file) { return file.data.id !== from.data.id; }); + // Use object identity: .done() replaces from.data in-place (MOVE op), + // making ID-based comparison unreliable by the time .always() runs. + tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function (file) { return file !== originalFrom; }); from.inProgress = false; if (notRenameOp && !tb.pendingReadyFiles.length){ doSyncMove(tb, to.data.provider); diff --git a/website/templates/emails/file_operation_failed.html.mako b/website/templates/emails/file_operation_failed.html.mako index 16adf99fc6e..e7ab9105d15 100644 --- a/website/templates/emails/file_operation_failed.html.mako +++ b/website/templates/emails/file_operation_failed.html.mako @@ -72,7 +72,7 @@ max_size_bytes = ei.get('max_size') or 0 max_size_mb = max_size_bytes // (1024 * 1024) if max_size_bytes else 0 %> - An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'} because the following file(s) exceed the maximum allowed file size% if max_size_mb: of ${max_size_mb} MB% endif: + An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'} because the following file(s) exceed the maximum allowed file size${' of {} MB'.format(max_size_mb) if max_size_mb else ''}:
      % for f in oversized_files:
    • ${f.get('name', 'Unknown file')} (${'{:.1f}'.format(f.get('size', 0) / (1024 * 1024))} MB)
      • From 66bbbd65d7a593333c464f36f60f778066920c6f Mon Sep 17 00:00:00 2001 From: hcphat Date: Wed, 17 Jun 2026 18:49:34 +0700 Subject: [PATCH 07/41] =?UTF-8?q?ref=202.2.=E3=82=A2=E3=83=89=E3=82=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AB?= =?UTF-8?q?=E7=B5=A1=E3=82=80=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E6=99=82=E3=81=AE=E3=82=AF=E3=82=AA=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=AE=A1=E7=90=86:=20Fix=20issue=20version=20incorrect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/osfstorage/models.py | 1 + addons/osfstorage/tests/test_models.py | 47 ++++++++++++++++++++++ osf/models/files.py | 6 ++- osf_tests/test_files.py | 54 +++++++++++++++++++++++++- 4 files changed, 105 insertions(+), 3 deletions(-) diff --git a/addons/osfstorage/models.py b/addons/osfstorage/models.py index 8786699ea9d..53813d82a72 100644 --- a/addons/osfstorage/models.py +++ b/addons/osfstorage/models.py @@ -361,6 +361,7 @@ def serialize(self, include_full=None, version=None): earliest_version = self.versions.order_by('created').first() ret.update({ 'version': self.versions.count(), + 'size': version.size if version else ret.get('size'), 'md5': version.metadata.get('md5') if version else None, 'sha256': version.metadata.get('sha256') if version else None, 'sha512': version.metadata.get('sha512') if version else None, diff --git a/addons/osfstorage/tests/test_models.py b/addons/osfstorage/tests/test_models.py index 27d81fc5b9b..fbaa3eb492e 100644 --- a/addons/osfstorage/tests/test_models.py +++ b/addons/osfstorage/tests/test_models.py @@ -137,6 +137,53 @@ def test_serialize(self): u'sha512': None, }) + def test_serialize_size_by_version(self): + file = OsfStorageFile(name='versioned_file.txt', target=self.node_settings.owner) + file.save() + + version1 = file.create_version( + self.user, + { + u'service': u'cloud', + settings.WATERBUTLER_RESOURCE: u'osf', + u'object': u'aaa111', + }, { + u'size': 1000, + u'contentType': u'text/plain', + }) + + version2 = file.create_version( + self.user, + { + u'service': u'cloud', + settings.WATERBUTLER_RESOURCE: u'osf', + u'object': u'bbb222', + }, { + u'size': 5000, + u'contentType': u'text/plain', + }) + + # Default serialize (no version arg) returns latest version's size + result_default = file.serialize() + assert_equal(result_default[u'size'], 5000) + assert_equal(result_default[u'version'], 2) + assert_equal(result_default[u'modified'], version2.created.isoformat()) + assert_equal(result_default[u'created'], version1.created.isoformat()) + + # Serialize with version=1 returns version 1's size + result_v1 = file.serialize(version=1) + assert_equal(result_v1[u'size'], 1000) + assert_equal(result_v1[u'version'], 2) + assert_equal(result_v1[u'modified'], version1.created.isoformat()) + assert_equal(result_v1[u'created'], version1.created.isoformat()) + + # Serialize with version=2 returns version 2's size + result_v2 = file.serialize(version=2) + assert_equal(result_v2[u'size'], 5000) + assert_equal(result_v2[u'version'], 2) + assert_equal(result_v2[u'modified'], version2.created.isoformat()) + assert_equal(result_v2[u'created'], version1.created.isoformat()) + def test_get_child_by_name(self): child = self.node_settings.get_root().append_file('Test') assert_equal(child, self.node_settings.get_root().find_child_by_name('Test')) diff --git a/osf/models/files.py b/osf/models/files.py index a53cc81f0ef..cdb8d42d838 100644 --- a/osf/models/files.py +++ b/osf/models/files.py @@ -557,7 +557,7 @@ def update(self, revision, data, user=None, save=True): return version def serialize(self): - newest_version = self.versions.all().last() + newest_version = self.versions.all().first() if not newest_version: return dict(self._serialize(), **{ @@ -571,6 +571,8 @@ def serialize(self): 'locked': self.locked._id if self.locked else None, }) + oldest_version = self.versions.all().last() + return dict(self._serialize(), **{ 'size': newest_version.size, 'downloads': self.get_download_count(), @@ -579,7 +581,7 @@ def serialize(self): 'version': newest_version.identifier if newest_version else None, 'contentType': newest_version.content_type if newest_version else None, 'modified': newest_version.external_modified.isoformat() if newest_version.external_modified else None, - 'created': self.versions.all().first().external_modified.isoformat() if self.versions.all().first().external_modified else None, + 'created': oldest_version.external_modified.isoformat() if oldest_version and oldest_version.external_modified else None, }) def restore(self, recursive=True, parent=None, save=True, deleted_on=None): diff --git a/osf_tests/test_files.py b/osf_tests/test_files.py index c8adac024ba..e8cdb9caa79 100644 --- a/osf_tests/test_files.py +++ b/osf_tests/test_files.py @@ -1,4 +1,7 @@ +import datetime + import pytest +from django.utils import timezone from django.contrib.contenttypes.models import ContentType @@ -8,7 +11,8 @@ from osf_tests.factories import ( UserFactory, ProjectFactory, - RegionFactory + RegionFactory, + FileVersionFactory, ) pytestmark = pytest.mark.django_db @@ -92,3 +96,51 @@ def test_file_update_respects_region(project, user, create_test_file): ) assert new_region != original_region assert new_version.region == new_region + + +def test_file_serialize_no_versions(project): + from addons.googledrive.models import GoogleDriveFile + + # GoogleDriveFile uses File.serialize directly (no override) + gd_file = GoogleDriveFile(name='empty.txt', target=project) + gd_file.save() + + result = gd_file.serialize() + + # newest_version is None → all version-derived fields are None + assert result['size'] is None + assert result['version'] is None + assert result['modified'] is None + assert result['created'] is None + assert result['contentType'] is None + + +def test_file_serialize_newest_version(project): + from addons.googledrive.models import GoogleDriveFile + + # GoogleDriveFile uses File.serialize directly (no override) + gd_file = GoogleDriveFile(name='multi_version.txt', target=project) + gd_file.save() + + # Create v1 first (older DB timestamp), smaller size + older_time = timezone.now() - datetime.timedelta(days=1) + v1 = FileVersionFactory(identifier='1', size=1000) + v1.external_modified = older_time + v1.save() + gd_file.add_version(v1) + + # Create v2 second (newer DB timestamp), larger size + newer_time = timezone.now() + v2 = FileVersionFactory(identifier='2', size=5000) + v2.external_modified = newer_time + v2.save() + gd_file.add_version(v2) + + result = gd_file.serialize() + + # newest_version = versions.all().first() ordered by -created → v2 + assert result['size'] == 5000 + assert result['version'] == '2' + assert result['modified'] == newer_time.isoformat() + # created comes from oldest_version (versions.all().last()) → v1 + assert result['created'] == older_time.isoformat() From ba196e1b305bafd5d6b97b9ac66f8acdc06fa20f Mon Sep 17 00:00:00 2001 From: hcphat Date: Wed, 24 Jun 2026 15:10:27 +0700 Subject: [PATCH 08/41] =?UTF-8?q?ref=202.2.=E3=82=A2=E3=83=89=E3=82=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AB?= =?UTF-8?q?=E7=B5=A1=E3=82=80=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E6=99=82=E3=81=AE=E3=82=AF=E3=82=AA=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=AE=A1=E7=90=86:=20Update=20script=20handle=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migration/migrate_fix_fileinfo_quota.py | 274 +++++++----------- 1 file changed, 101 insertions(+), 173 deletions(-) diff --git a/scripts/migration/migrate_fix_fileinfo_quota.py b/scripts/migration/migrate_fix_fileinfo_quota.py index 406ceeef355..73464b870d5 100644 --- a/scripts/migration/migrate_fix_fileinfo_quota.py +++ b/scripts/migration/migrate_fix_fileinfo_quota.py @@ -20,13 +20,23 @@ 4. Calls update_user_used_quota(user) for each affected user to re-sync UserQuota.used from the actual FileInfo totals. Storage type is determined per-project (NII_STORAGE or CUSTOM_STORAGE). -5. Exports a CSV report of all processed records, including skipped files. +5. Exports a CSV report of all processed records. -Database Transactions: - All database updates (FileInfo insertions/updates and UserQuota recalculations) - are wrapped in a single database transaction (transaction.atomic). If the script - crashes or encounters an unhandled exception during the fix process, all changes - are rolled back to ensure database consistency. +Error Handling (fail-fast): + The script fails fast. As soon as it hits a record that cannot be fully fixed — + its size cannot be determined, or its project creator / storage_type cannot be + resolved — or any unexpected error, it logs THAT single record and raises + immediately, so an administrator can fix that record and re-run. It does NOT + scan the whole dataset to collect and print a list of every problem. This + behaviour is identical in dry-run and in a real run. + + All DB updates run inside a single transaction (transaction.atomic). If an error + is raised part-way through a real run, every change is rolled back and the + database is left untouched (no partially-applied fixes). + + Note on file size: size is read from the latest FileVersion. A genuine + file_size == 0 is a valid size and a FileInfo record with file_size=0 is + created/updated; only an indeterminable size (None) is treated as an error. Usage: # Fix and export results to CSV @@ -38,7 +48,6 @@ python -m scripts.migration.migrate_fix_fileinfo_quota --dry --output preview.csv """ -import sys import os import csv import logging @@ -55,11 +64,10 @@ from django.db.models import Q from addons.osfstorage.models import OsfStorageFile -from osf.models import FileInfo, OSFUser, UserQuota, ProjectStorageType +from osf.models import FileInfo, OSFUser from website.util.quota import update_user_used_quota, get_project_storage_type from scripts import utils as script_utils from django.contrib.contenttypes.models import ContentType -from osf.models import Guid logger = logging.getLogger(__name__) @@ -69,7 +77,7 @@ # fileinfo__id : _id (hex string) of the FileInfo record inserted/updated (empty in dry_run) # fileinfo_file_id: integer PK of BaseFileNode — the value stored as file_id (FK) in the FileInfo table # fileinfo_created: True = new FileInfo row INSERT-ed, False = existing row UPDATE-d (empty in dry_run) - # action : would_update | created | updated | updated_no_quota | skipped + # action : would_update (dry-run) | created | updated # quota_recalculated: True if quota was recalculated for this file's creator 'osf_file_id', 'file_name', 'fileinfo_id', 'fileinfo__id', 'fileinfo_file_id', @@ -145,8 +153,10 @@ def _bulk_get_latest_version_sizes(osf_file_ids): def scan_broken_files(): """Scan for OsfStorageFile nodes with missing or zero-size FileInfo. - Returns a list of dicts with scan results (no DB writes). - Skipped files are also included with action='skipped' for full audit visibility. + Fails fast: the moment a record is found that cannot be fully fixed (size cannot + be determined, or its project creator / storage_type cannot be resolved), this + raises so the administrator can fix that one record and re-run. No DB writes. + Returns a list of fully-fixable row dicts when every record is OK. """ broken_files = list(get_broken_file_nodes()) total = len(broken_files) @@ -158,30 +168,18 @@ def scan_broken_files(): size_map = _bulk_get_latest_version_sizes(file_ids) rows = [] - for idx, osf_file in enumerate(broken_files, 1): + for osf_file in broken_files: size = size_map.get(osf_file.id) - if size is None or size <= 0: - logger.warning( - '[{}/{}] Skipping {} (_id={}) — could not determine size from FileVersion.'.format( - idx, total, osf_file.name, osf_file._id - ) + # Fail fast: the size must be determinable. (A genuine file_size == 0 is a + # valid size and is NOT an error — only an indeterminable size is.) + if size is None: + raise RuntimeError( + 'File {} (_id={}) — could not determine size from any FileVersion. ' + 'Fix this record and re-run.'.format(osf_file.name, osf_file._id) ) - # Include skipped files in CSV for full audit trail - rows.append({ - 'osf_file_id': osf_file._id, - 'fileinfo_file_id': osf_file.id, - 'file_name': osf_file.name, - 'file_size_bytes': size if size is not None else '', - 'project_guid': '', - 'creator_id': '', - 'storage_type': None, - 'action': 'skipped', - 'quota_recalculated': False, - 'osf_file': osf_file, - }) - continue + # Resolve the project target, then its creator / guid / storage_type. project_guid = '' creator_id = '' storage_type_value = None @@ -202,30 +200,33 @@ def scan_broken_files(): creator = getattr(target, 'creator', None) if creator: creator_id = creator._id - # Determine storage type for this project storage_type_value = get_project_storage_type(target) - else: - # Target object not found — try Guid table as last resort - if osf_file.target_content_type_id and osf_file.target_object_id: - ct = ContentType.objects.get_for_id(osf_file.target_content_type_id) - guid_obj = Guid.objects.filter( - content_type=ct, - object_id=osf_file.target_object_id - ).first() - if guid_obj: - project_guid = guid_obj._id - logger.warning( - 'File {} target not found (target_content_type_id={}, target_object_id={}). ' - 'Resolved project_guid={}'.format( - osf_file._id, osf_file.target_content_type_id, osf_file.target_object_id, project_guid - ) - ) except Exception as e: - logger.warning('Could not resolve creator/guid for file {}: {}'.format(osf_file._id, e)) + # Fail fast: do not swallow unexpected errors. + logger.error('Could not resolve target/creator for file {}: {} — aborting (fail fast).'.format( + osf_file._id, e + )) + raise + + # Fail fast: creator and storage_type must be resolvable to recalculate quota. + if not creator_id: + raise RuntimeError( + 'File {} (_id={}, project_guid={}) — creator could not be resolved ' + '(target missing or has no creator). Fix this record and re-run.'.format( + osf_file.name, osf_file._id, project_guid + ) + ) + if storage_type_value is None: + raise RuntimeError( + 'File {} (_id={}, project_guid={}) — project storage_type could not be ' + 'determined. Fix this record and re-run.'.format( + osf_file.name, osf_file._id, project_guid + ) + ) rows.append({ 'osf_file_id': osf_file._id, # hex _id string of OsfStorageFile - 'fileinfo_file_id': osf_file.id, # integer PK stored as file_id FK in FileInfo table + 'fileinfo_file_id': osf_file.id, # integer PK stored as file_id FK in FileInfo table 'file_name': osf_file.name, 'file_size_bytes': size, 'project_guid': project_guid, @@ -234,46 +235,37 @@ def scan_broken_files(): 'osf_file': osf_file, }) - eligible = sum(1 for r in rows if r.get('action') != 'skipped') - logger.info('Scan complete. {} record(s) eligible for fix, {} skipped.'.format( - eligible, total - eligible - )) + logger.info('Scan complete. {} record(s) ready to fix.'.format(len(rows))) return rows def print_results(rows): """Display scanned records in a formatted summary table.""" - eligible = [r for r in rows if r.get('action') != 'skipped'] - skipped = [r for r in rows if r.get('action') == 'skipped'] - - if not eligible and not skipped: + if not rows: print(' No broken FileInfo records found.') return - if eligible: - print('\n' + '-' * 160) + def _disp(value): + # Display helper: never let None (or any value) break str formatting. + return '' if value is None else str(value) + + print('\n' + '-' * 160) + print(' {:<26} {:<16} {:<22} {:<18} {:<16} {:<14} {:<18}'.format( + 'OSF_FILE_ID (_id)', 'FILEINFO_FILE_ID', 'FILE_NAME', 'PROJECT_GUID', 'CREATOR_ID', 'STORAGE_TYPE', 'SIZE (bytes)' + )) + print('-' * 160) + for r in rows: print(' {:<26} {:<16} {:<22} {:<18} {:<16} {:<14} {:<18}'.format( - 'OSF_FILE_ID (_id)', 'FILEINFO_FILE_ID', 'FILE_NAME', 'PROJECT_GUID', 'CREATOR_ID', 'STORAGE_TYPE', 'SIZE (bytes)' + _disp(r.get('osf_file_id')), + _disp(r.get('fileinfo_file_id')), + _disp(r.get('file_name'))[:20], + _disp(r.get('project_guid')), + _disp(r.get('creator_id')), + _disp(r.get('storage_type')), + _disp(r.get('file_size_bytes')), )) - print('-' * 160) - for r in eligible: - print(' {:<26} {:<16} {:<22} {:<18} {:<16} {:<14} {:<18}'.format( - r['osf_file_id'], - r['fileinfo_file_id'], - (r['file_name'] or '')[:20], - r['project_guid'], - r['creator_id'], - str(r.get('storage_type') if r.get('storage_type') is not None else ''), - r['file_size_bytes'], - )) - print('-' * 160) - print(' Total: {} record(s) would be updated.\n'.format(len(eligible))) - - if skipped: - print(' Skipped {} record(s) (no valid FileVersion size):'.format(len(skipped))) - for r in skipped: - print(' - {} ({})'.format(r['osf_file_id'], r['file_name'])) - print() + print('-' * 160) + print(' Total: {} record(s) would be updated.\n'.format(len(rows))) # --- Updating Operations --- @@ -298,56 +290,23 @@ def apply_fixes(rows, dry_run=False): def _apply_fixes_internal(rows, dry_run=False): - """Internal helper to apply fixes, called inside/outside transaction.""" + """Internal helper to apply fixes, called inside/outside transaction. + + Every row here is already fully fixable (scan_broken_files fails fast otherwise), + so each row gets a FileInfo update and contributes to a UserQuota recalculation. + """ result_rows = [] affected_user_storage = set() # set of (creator_id, storage_type) fixed = 0 - skipped = 0 for row in rows: - if row.get('action') == 'skipped': - result_rows.append({ - 'osf_file_id': row['osf_file_id'], - 'fileinfo_file_id': row['fileinfo_file_id'], - 'file_name': row['file_name'], - 'file_size_bytes': row['file_size_bytes'], - 'project_guid': row['project_guid'], - 'creator_id': row['creator_id'], - 'storage_type': row['storage_type'], - 'fileinfo_id': '', - 'fileinfo__id': '', - 'fileinfo_created': '', - 'action': 'skipped', - 'quota_recalculated': False, - 'dry_run': dry_run, - }) - skipped += 1 - continue - - # Optimize: reuse osf_file object from scan, avoiding re-querying + # scan stores the osf_file object on the row; fall back to a lookup just in case. osf_file = row.get('osf_file') if osf_file is None: osf_file = OsfStorageFile.objects.filter(_id=row['osf_file_id']).first() - if osf_file is None: - logger.warning('File {} not found, skipping.'.format(row['osf_file_id'])) - result_rows.append({ - 'osf_file_id': row['osf_file_id'], - 'fileinfo_file_id': row['fileinfo_file_id'], - 'file_name': row['file_name'], - 'file_size_bytes': row['file_size_bytes'], - 'project_guid': row['project_guid'], - 'creator_id': row['creator_id'], - 'storage_type': row['storage_type'], - 'fileinfo_id': '', - 'fileinfo__id': '', - 'fileinfo_created': '', - 'action': 'skipped', - 'quota_recalculated': False, - 'dry_run': dry_run, - }) - skipped += 1 - continue + # Fail fast: a scanned row must resolve to a file. + raise RuntimeError('File {} not found when applying fix.'.format(row['osf_file_id'])) fileinfo_id = '' fileinfo__id = '' @@ -361,23 +320,11 @@ def _apply_fixes_internal(rows, dry_run=False): fileinfo__id = fileinfo_obj._id # hex _id string of FileInfo fileinfo_created = created # True = INSERT, False = UPDATE - # Determine whether quota will actually be recalculated - creator_id = row.get('creator_id', '') - storage_type = row.get('storage_type', None) - quota_will_recalculate = bool(creator_id and storage_type is not None) + creator_id = row['creator_id'] + storage_type = row['storage_type'] + affected_user_storage.add((creator_id, storage_type)) - if quota_will_recalculate: - # Track (creator_id, storage_type) pair - affected_user_storage.add((creator_id, storage_type)) - - # Use distinct action when creator_id is missing - if dry_run: - action = 'would_update' - elif not quota_will_recalculate: - # FileInfo was fixed but quota cannot be recalculated (no creator resolved) - action = 'updated_no_quota' - else: - action = 'created' if fileinfo_created else 'updated' + action = 'would_update' if dry_run else ('created' if fileinfo_created else 'updated') logger.info('{} {} (osf_file_id={}, fileinfo_id={}, fileinfo__id={}, fileinfo_file_id={}, created={}) file_size={} bytes storage_type={}'.format( '[DRY RUN]' if dry_run else 'Fixed:', @@ -386,14 +333,6 @@ def _apply_fixes_internal(rows, dry_run=False): fileinfo_created, row['file_size_bytes'], storage_type, )) - if action == 'updated_no_quota': - logger.warning( - 'File {} FileInfo fixed but creator_id is empty — UserQuota NOT recalculated. ' - 'Manual review required. project_guid={}'.format( - row['osf_file_id'], row.get('project_guid', '') - ) - ) - result_rows.append({ 'osf_file_id': row['osf_file_id'], 'fileinfo_file_id': row['fileinfo_file_id'], @@ -411,9 +350,7 @@ def _apply_fixes_internal(rows, dry_run=False): }) fixed += 1 - logger.info('FileInfo backfill complete. fixed={}, skipped={} (dry={})'.format( - fixed, skipped, dry_run - )) + logger.info('FileInfo backfill complete. fixed={} (dry={})'.format(fixed, dry_run)) # --- Recalculate UserQuota.used for all affected (user, storage_type) pairs --- logger.info('Recalculating UserQuota.used for {} unique (user, storage_type) pair(s).'.format( @@ -421,19 +358,20 @@ def _apply_fixes_internal(rows, dry_run=False): )) quota_updated = 0 - quota_errors = 0 - # Track which creator_ids had successful quota recalculation recalculated_user_storage = set() for user_id, storage_type in sorted(affected_user_storage): try: user = OSFUser.load(user_id) if user is None: - logger.warning('User {} not found, skipping quota recalc.'.format(user_id)) - continue + # Fail fast: a creator_id resolved during scan must load here. + raise RuntimeError( + 'User {} (storage_type={}) could not be loaded for quota recalc.'.format( + user_id, storage_type + ) + ) if not dry_run: - # Update quota for the actual storage_type of the project update_user_used_quota(user, storage_type=storage_type, is_recalculating_quota=True) logger.info('{} Recalculated quota for user {} (storage_type={})'.format( @@ -442,23 +380,18 @@ def _apply_fixes_internal(rows, dry_run=False): quota_updated += 1 recalculated_user_storage.add((user_id, storage_type)) except Exception as e: - logger.error('Failed to recalculate quota for user {} (storage_type={}): {}'.format( - user_id, storage_type, e - )) - quota_errors += 1 + # Fail fast: re-raise so transaction.atomic() rolls back ALL FileInfo updates. + logger.error('Failed to recalculate quota for user {} (storage_type={}): {} — ' + 'rolling back transaction (fail fast).'.format(user_id, storage_type, e)) + raise - logger.info('Quota recalculation complete. updated={}, errors={} (dry={})'.format( - quota_updated, quota_errors, dry_run - )) + logger.info('Quota recalculation complete. updated={} (dry={})'.format(quota_updated, dry_run)) # Update quota_recalculated flag in result rows for row in result_rows: - creator_id = row.get('creator_id', '') - storage_type = row.get('storage_type', None) - if dry_run and creator_id and storage_type is not None: - # In dry-run mode, mark as "would recalculate" + if dry_run: row['quota_recalculated'] = 'would_recalculate' - elif (creator_id, storage_type) in recalculated_user_storage: + elif (row['creator_id'], row['storage_type']) in recalculated_user_storage: row['quota_recalculated'] = True return result_rows @@ -473,13 +406,7 @@ def save_result_csv(result_rows, output_path): writer.writeheader() writer.writerows(result_rows) total = len(result_rows) - skipped = sum(1 for r in result_rows if r.get('action') == 'skipped') - no_quota = sum(1 for r in result_rows if r.get('action') == 'updated_no_quota') print('\n Exported {} result(s) to: {}'.format(total, output_path)) - if skipped: - print(' - {} skipped (no valid FileVersion size)'.format(skipped)) - if no_quota: - print(' - {} updated_no_quota (FileInfo fixed but quota NOT recalculated — manual review needed)'.format(no_quota)) # --- Main --- @@ -522,7 +449,8 @@ def main(): # 2. Print summary table print_results(rows) - # 3. Apply fixes (or simulate in dry-run mode) + # 3. Apply fixes (or simulate in dry-run mode). scan_broken_files() has already + # failed fast on any unfixable record, so every row here is fully fixable. result_rows = apply_fixes(rows, dry_run=args.dry) # 4. Export results to CSV From 106c84b81e20695c73a5f349cbd4586ec6a71910 Mon Sep 17 00:00:00 2001 From: Phat Nguyen Date: Wed, 12 Aug 2026 18:36:08 +0700 Subject: [PATCH 09/41] =?UTF-8?q?Ref:=20=E6=A9=9F=E9=96=A2=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E8=80=85=E6=A9=9F=E8=83=BDAPI=E3=81=AE=E9=81=A9?= =?UTF-8?q?=E5=88=87=E3=81=AA=E6=A8=A9=E9=99=90=E4=BB=98=E4=B8=8E:=20Add?= =?UTF-8?q?=20a=20permission=20check=20step=20for=20some=20admin=20endpoin?= =?UTF-8?q?ts=20and=20update=20UT.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- admin/base/urls.py | 106 ++++++++++-------- admin/base/utils.py | 19 ++++ admin/collection_providers/views.py | 8 +- admin/common_auth/views.py | 10 +- admin/management/views.py | 18 ++- admin/preprint_providers/views.py | 8 +- admin/quota_recalc/views.py | 3 + admin/rdm_statistics/views.py | 24 +++- admin/registration_providers/views.py | 10 +- .../collection_providers/test_views.py | 45 ++++++++ admin_tests/common_auth/test_views.py | 49 +++++++- admin_tests/management/__init__.py | 0 admin_tests/management/test_views.py | 70 ++++++++++++ admin_tests/preprint_providers/test_views.py | 45 ++++++++ admin_tests/quota_recalc/test_views.py | 69 +++++++++++- admin_tests/rdm_statistics/test_views.py | 79 ++++++++++++- .../registration_providers/test_views.py | 79 +++++++++++++ 17 files changed, 566 insertions(+), 76 deletions(-) create mode 100644 admin_tests/management/__init__.py create mode 100644 admin_tests/management/test_views.py diff --git a/admin/base/urls.py b/admin/base/urls.py index c63c9d20fbd..85946577196 100644 --- a/admin/base/urls.py +++ b/admin/base/urls.py @@ -1,62 +1,70 @@ from django.conf.urls import include, url from django.contrib import admin -from admin.base.settings import ADMIN_BASE, DEBUG +from admin.base.settings import ADMIN_BASE, DEBUG, ENABLE_LOGIN_FORM from admin.base import views base_pattern = '^{}'.format(ADMIN_BASE) +admin_urls = [ + url(r'^$', views.home, name='home'), + url(r'^admin/', admin.site.urls), + url(r'^asset_files/', include('admin.asset_files.urls', namespace='asset_files')), + url(r'^banners/', include('admin.banners.urls', namespace='banners')), + url(r'^brands/', include('admin.brands.urls', namespace='brands')), + url(r'^spam/', include('admin.spam.urls', namespace='spam')), + url(r'^institutions/', include('admin.institutions.urls', namespace='institutions')), + url(r'^entitlements/', include('admin.entitlements.urls', namespace='entitlements')), + url(r'^quota_recalc/', include('admin.quota_recalc.urls', namespace='quota_recalc')), + url(r'^preprint_providers/', include('admin.preprint_providers.urls', namespace='preprint_providers')), + url(r'^collection_providers/', include('admin.collection_providers.urls', namespace='collection_providers')), + url(r'^registration_providers/', include('admin.registration_providers.urls', namespace='registration_providers')), + url(r'^account/', include('admin.common_auth.urls', namespace='auth')), +] + +if ENABLE_LOGIN_FORM: + # password recovery/reset only makes sense while password-based login itself is enabled + admin_urls.append(url(r'^password/', include('password_reset.urls'))) + +admin_urls += [ + url(r'^nodes/', include('admin.nodes.urls', namespace='nodes')), + url(r'^preprints/', include('admin.preprints.urls', namespace='preprints')), + url(r'^subjects/', include('admin.subjects.urls', namespace='subjects')), + url(r'^users/', include('admin.users.urls', namespace='users')), + url(r'^user-emails/', include('admin.user_emails.urls', namespace='user-emails')), + url(r'^maintenance/', include('admin.maintenance.urls', namespace='maintenance')), + url(r'^meetings/', include('admin.meetings.urls', + namespace='meetings')), + url(r'^metrics/', include('admin.metrics.urls', + namespace='metrics')), + url(r'^desk/', include('admin.desk.urls', + namespace='desk')), + url(r'^osf_groups/', include('admin.osf_groups.urls', namespace='osf_groups')), + url(r'^management/', include('admin.management.urls', namespace='management')), + url(r'^announcement/', include('admin.rdm_announcement.urls', namespace='announcement')), + url(r'^addons/', include('admin.rdm_addons.urls', namespace='addons')), + url(r'^oauth/', include('admin.rdm_addons.oauth.urls', namespace='oauth')), + url(r'^statistics/', include('admin.rdm_statistics.urls', namespace='statistics')), + url(r'^timestampadd/', include('admin.rdm_timestampadd.urls', namespace='timestampadd')), + url(r'^keymanagement/', include('admin.rdm_keymanagement.urls', namespace='keymanagement')), + url(r'^timestampsettings/', include('admin.rdm_timestampsettings.urls', namespace='timestampsettings')), + url(r'^custom_storage_location/', include('admin.rdm_custom_storage_location.urls', namespace='custom_storage_location')), + url(r'^institutional_storage_quota_control/', include('admin.institutional_storage_quota_control.urls', + namespace='institutional_storage_quota_control')), + url(r'^metadata/', include('admin.rdm_metadata.urls', namespace='metadata')), + url(r'^user_identification_information/', + include('admin.user_identification_information.urls', namespace='user_identification_information')), + url(r'^user_identification_information_admin/', + include('admin.user_identification_information_admin.urls', namespace='user_identification_information_admin')), + url(r'^project_limit_number/', include('admin.project_limit_number.urls', namespace='project_limit_number')), + url(r'^rdm_workflow/', include('admin.rdm_workflow.urls', namespace='rdm_workflow')), + url(r'^loa/', include('admin.loa.urls', namespace='loa')), +] + urlpatterns = [ ### ADMIN ### url( base_pattern, - include([ - url(r'^$', views.home, name='home'), - url(r'^admin/', admin.site.urls), - url(r'^asset_files/', include('admin.asset_files.urls', namespace='asset_files')), - url(r'^banners/', include('admin.banners.urls', namespace='banners')), - url(r'^brands/', include('admin.brands.urls', namespace='brands')), - url(r'^spam/', include('admin.spam.urls', namespace='spam')), - url(r'^institutions/', include('admin.institutions.urls', namespace='institutions')), - url(r'^entitlements/', include('admin.entitlements.urls', namespace='entitlements')), - url(r'^quota_recalc/', include('admin.quota_recalc.urls', namespace='quota_recalc')), - url(r'^preprint_providers/', include('admin.preprint_providers.urls', namespace='preprint_providers')), - url(r'^collection_providers/', include('admin.collection_providers.urls', namespace='collection_providers')), - url(r'^registration_providers/', include('admin.registration_providers.urls', namespace='registration_providers')), - url(r'^account/', include('admin.common_auth.urls', namespace='auth')), - url(r'^password/', include('password_reset.urls')), - url(r'^nodes/', include('admin.nodes.urls', namespace='nodes')), - url(r'^preprints/', include('admin.preprints.urls', namespace='preprints')), - url(r'^subjects/', include('admin.subjects.urls', namespace='subjects')), - url(r'^users/', include('admin.users.urls', namespace='users')), - url(r'^user-emails/', include('admin.user_emails.urls', namespace='user-emails')), - url(r'^maintenance/', include('admin.maintenance.urls', namespace='maintenance')), - url(r'^meetings/', include('admin.meetings.urls', - namespace='meetings')), - url(r'^metrics/', include('admin.metrics.urls', - namespace='metrics')), - url(r'^desk/', include('admin.desk.urls', - namespace='desk')), - url(r'^osf_groups/', include('admin.osf_groups.urls', namespace='osf_groups')), - url(r'^management/', include('admin.management.urls', namespace='management')), - url(r'^announcement/', include('admin.rdm_announcement.urls', namespace='announcement')), - url(r'^addons/', include('admin.rdm_addons.urls', namespace='addons')), - url(r'^oauth/', include('admin.rdm_addons.oauth.urls', namespace='oauth')), - url(r'^statistics/', include('admin.rdm_statistics.urls', namespace='statistics')), - url(r'^timestampadd/', include('admin.rdm_timestampadd.urls', namespace='timestampadd')), - url(r'^keymanagement/', include('admin.rdm_keymanagement.urls', namespace='keymanagement')), - url(r'^timestampsettings/', include('admin.rdm_timestampsettings.urls', namespace='timestampsettings')), - url(r'^custom_storage_location/', include('admin.rdm_custom_storage_location.urls', namespace='custom_storage_location')), - url(r'^institutional_storage_quota_control/', include('admin.institutional_storage_quota_control.urls', - namespace='institutional_storage_quota_control')), - url(r'^metadata/', include('admin.rdm_metadata.urls', namespace='metadata')), - url(r'^user_identification_information/', - include('admin.user_identification_information.urls', namespace='user_identification_information')), - url(r'^user_identification_information_admin/', - include('admin.user_identification_information_admin.urls', namespace='user_identification_information_admin')), - url(r'^project_limit_number/', include('admin.project_limit_number.urls', namespace='project_limit_number')), - url(r'^rdm_workflow/', include('admin.rdm_workflow.urls', namespace='rdm_workflow')), - url(r'^loa/', include('admin.loa.urls', namespace='loa')), - ]), + include(admin_urls), ), ] diff --git a/admin/base/utils.py b/admin/base/utils.py index 6ee8780be71..4f063bbbea3 100644 --- a/admin/base/utils.py +++ b/admin/base/utils.py @@ -1,6 +1,8 @@ """ Utility functions and classes """ +from functools import wraps + from osf.models import Subject, NodeLicense, Brand from django.core.exceptions import ValidationError, PermissionDenied @@ -35,6 +37,23 @@ def osf_staff_check(user): return user.is_authenticated and user.is_staff +def osf_superuser_check(user): + return user.is_authenticated and user.is_superuser + + +def superuser_required(view_func): + """Function-view decorator that returns 403 (PermissionDenied) instead of + redirecting to login, unlike django.contrib.auth.decorators.user_passes_test.""" + + @wraps(view_func) + def wrapped_view(request, *args, **kwargs): + if not osf_superuser_check(request.user): + raise PermissionDenied + return view_func(request, *args, **kwargs) + + return wrapped_view + + def get_subject_rules(subjects_selected): """ Take a list of subjects, and parse them into rules consistent with preprpint provider diff --git a/admin/collection_providers/views.py b/admin/collection_providers/views.py index cafb9476f9e..4cfd366d990 100644 --- a/admin/collection_providers/views.py +++ b/admin/collection_providers/views.py @@ -4,7 +4,7 @@ from django.core import serializers from django.core.exceptions import ValidationError from django.urls import reverse_lazy -from django.shortcuts import redirect +from django.shortcuts import redirect, get_object_or_404 from django.views.generic import View, CreateView, ListView, DetailView, UpdateView, DeleteView, TemplateView from django.contrib import messages from django.contrib.auth.mixins import PermissionRequiredMixin @@ -228,12 +228,14 @@ def get_object(self, queryset=None): return CollectionProvider.objects.get(id=self.kwargs['collection_provider_id']) -class CannotDeleteProvider(TemplateView): +class CannotDeleteProvider(PermissionRequiredMixin, TemplateView): + permission_required = 'osf.delete_collectionprovider' + raise_exception = True template_name = 'collection_providers/cannot_delete.html' def get_context_data(self, **kwargs): context = super(CannotDeleteProvider, self).get_context_data(**kwargs) - context['provider'] = CollectionProvider.objects.get(id=self.kwargs['collection_provider_id']) + context['provider'] = get_object_or_404(CollectionProvider, id=self.kwargs['collection_provider_id']) return context diff --git a/admin/common_auth/views.py b/admin/common_auth/views.py index d2dc3993090..dd03b30dedb 100644 --- a/admin/common_auth/views.py +++ b/admin/common_auth/views.py @@ -39,10 +39,12 @@ def dispatch(self, request, *args, **kwargs): return super(LoginView, self).dispatch(request, *args, **kwargs) def form_valid(self, form): - user = authenticate( - username=form.cleaned_data.get('email').strip(), - password=form.cleaned_data.get('password').strip() - ) + user = None + if ENABLE_LOGIN_FORM: + user = authenticate( + username=form.cleaned_data.get('email').strip(), + password=form.cleaned_data.get('password').strip() + ) if user is not None: login(self.request, user) else: diff --git a/admin/management/views.py b/admin/management/views.py index fffd1bbef4f..b1522991ad2 100644 --- a/admin/management/views.py +++ b/admin/management/views.py @@ -1,15 +1,29 @@ +from django.contrib.auth.mixins import UserPassesTestMixin from django.views.generic import TemplateView, View from osf.management.commands.manage_switch_flags import manage_waffle from django.core.urlresolvers import reverse from django.shortcuts import redirect -class ManagementCommands(TemplateView): +from admin.rdm.utils import RdmPermissionMixin + +class ManagementCommands(RdmPermissionMixin, UserPassesTestMixin, TemplateView): """ Basic form to trigger various management commands """ template_name = 'management/commands.html' object_type = 'management' + raise_exception = True + + def test_func(self): + """check user permissions""" + return self.is_super_admin + + +class WaffleFlag(RdmPermissionMixin, UserPassesTestMixin, View): + raise_exception = True -class WaffleFlag(View): + def test_func(self): + """check user permissions""" + return self.is_super_admin def post(self, request, *args, **kwargs): manage_waffle() diff --git a/admin/preprint_providers/views.py b/admin/preprint_providers/views.py index 159a0a409f5..8d2ecbf6d29 100644 --- a/admin/preprint_providers/views.py +++ b/admin/preprint_providers/views.py @@ -14,7 +14,7 @@ from django.contrib import messages from django.contrib.auth.mixins import PermissionRequiredMixin from django.forms.models import model_to_dict -from django.shortcuts import redirect, render +from django.shortcuts import redirect, render, get_object_or_404 from admin.base import settings from admin.base.forms import ImportFileForm @@ -319,12 +319,14 @@ def get_object(self, queryset=None): return PreprintProvider.objects.get(id=self.kwargs['preprint_provider_id']) -class CannotDeleteProvider(TemplateView): +class CannotDeleteProvider(PermissionRequiredMixin, TemplateView): + permission_required = 'osf.delete_preprintprovider' + raise_exception = True template_name = 'preprint_providers/cannot_delete.html' def get_context_data(self, **kwargs): context = super(CannotDeleteProvider, self).get_context_data(**kwargs) - context['provider'] = PreprintProvider.objects.get(id=self.kwargs['preprint_provider_id']) + context['provider'] = get_object_or_404(PreprintProvider, id=self.kwargs['preprint_provider_id']) return context diff --git a/admin/quota_recalc/views.py b/admin/quota_recalc/views.py index 29484c3d559..540e1acef0d 100644 --- a/admin/quota_recalc/views.py +++ b/admin/quota_recalc/views.py @@ -1,6 +1,7 @@ from django.http import JsonResponse from django.db import transaction, IntegrityError +from admin.base.utils import superuser_required from addons.osfstorage.models import Region from api.base import settings as api_settings from osf.models import OSFUser, UserQuota, Node @@ -60,6 +61,7 @@ def calculate_quota(user): user_quota.used = used user_quota.save() +@superuser_required def all_users(request, **kwargs): c = 0 for osf_user in OSFUser.objects.exclude(deleted__isnull=False): @@ -70,6 +72,7 @@ def all_users(request, **kwargs): 'message': str(c) + ' users\' quota successfully recalculated!' }) +@superuser_required def user(request, guid, **kwargs): user = OSFUser.load(guid) if user is None: diff --git a/admin/rdm_statistics/views.py b/admin/rdm_statistics/views.py index d4522cf1a9c..262c8edce42 100644 --- a/admin/rdm_statistics/views.py +++ b/admin/rdm_statistics/views.py @@ -41,6 +41,7 @@ import seaborn as sns import pdfkit from admin.base import settings +from admin.base.utils import superuser_required from admin.rdm.utils import RdmPermissionMixin, get_dummy_institution from admin.rdm_addons import utils import logging @@ -484,7 +485,7 @@ def get(self, request, *args, **kwargs): response_json = json.dumps(self.stat_list) response = HttpResponse(response_json, content_type='application/json') # statistics mail send - send_stat_mail(request) + send_stat_mail_core(request) except Exception as err: response_hash = {'state': 'fail', 'error': str(err)} response_json = json.dumps(response_hash) @@ -596,8 +597,14 @@ def simple_auth(access_token): else: return False -def send_stat_mail(request, **kwargs): - """send statistics information email""" +def send_stat_mail_core(request, **kwargs): + """Send statistics information email. + + No authorization check here by design - this is the shared logic called + both by the send_stat_mail view (protected below) and internally by + GatherView.get(), which runs unauthenticated over localhost from + cron and is authorized separately via its own access token. + """ current_date = get_current_date() all_institutions = Institution.objects.order_by('id').all() all_staff_users = OSFUser.objects.filter(is_staff=True) @@ -630,6 +637,11 @@ def send_stat_mail(request, **kwargs): response = HttpResponse(response_json, content_type='application/json') return response +@superuser_required +def send_stat_mail(request, **kwargs): + """URL-exposed view for /statistics/test/mail/ - Integrated Admin only.""" + return send_stat_mail_core(request, **kwargs) + def send_error_mail(err): """send error email""" current_date = get_current_date() @@ -776,11 +788,15 @@ def approximate_size(size, a_kilobyte_is_1024_bytes=True): ### views or funcs for development and test ############################################ -class IndexView(TemplateView): +class IndexView(RdmPermissionMixin, UserPassesTestMixin, TemplateView): """index view of statistics module.""" template_name = 'rdm_statistics/index.html' raise_exception = True + def test_func(self): + """check user permissions""" + return self.is_authenticated and (self.is_super_admin or self.is_admin) + def find_bookmark_collection(self, user): collection = apps.get_model('osf.Collection') return collection.objects.get(creator=user, is_deleted=False, is_bookmark_collection=True) diff --git a/admin/registration_providers/views.py b/admin/registration_providers/views.py index 2466bfae63a..c576ed7db16 100644 --- a/admin/registration_providers/views.py +++ b/admin/registration_providers/views.py @@ -6,7 +6,7 @@ from django.core.exceptions import ValidationError from django.core.management import call_command from django.urls import reverse_lazy -from django.shortcuts import redirect +from django.shortcuts import redirect, get_object_or_404 from django.views.generic import View, CreateView, ListView, DetailView, UpdateView, DeleteView, TemplateView from django.contrib.auth.mixins import PermissionRequiredMixin from django.contrib import messages @@ -204,12 +204,14 @@ def get_context_data(self, *args, **kwargs): return super(DeleteRegistrationProvider, self).get_context_data(*args, **kwargs) -class CannotDeleteProvider(TemplateView): +class CannotDeleteProvider(PermissionRequiredMixin, TemplateView): + permission_required = 'osf.delete_registrationprovider' + raise_exception = True template_name = 'registration_providers/cannot_delete.html' def get_context_data(self, **kwargs): context = super(CannotDeleteProvider, self).get_context_data(**kwargs) - context['provider'] = RegistrationProvider.objects.get(id=self.kwargs['registration_provider_id']) + context['provider'] = get_object_or_404(RegistrationProvider, id=self.kwargs['registration_provider_id']) return context @@ -376,7 +378,7 @@ def get(self, request, *args, **kwargs): return redirect(reverse_lazy('registration_providers:detail', kwargs={'registration_provider_id': provider.id})) -class ChangeSchema(TemplateView): +class ChangeSchema(PermissionRequiredMixin, TemplateView): permission_required = 'osf.change_registrationprovider' template_name = 'registration_providers/change_schema.html' diff --git a/admin_tests/collection_providers/test_views.py b/admin_tests/collection_providers/test_views.py index ef054a2778f..1dd19e84801 100644 --- a/admin_tests/collection_providers/test_views.py +++ b/admin_tests/collection_providers/test_views.py @@ -1,6 +1,9 @@ import pytest from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser, Permission +from django.core.exceptions import PermissionDenied +from django.http import Http404 from osf_tests.factories import ( AuthUserFactory, @@ -93,3 +96,45 @@ def view(self, req, provider): view = setup_view(view, req) view.kwargs = {'collection_provider_id': provider.id} return view + + +@pytest.mark.urls('admin.base.urls') +class TestCannotDeleteProviderPermission: + """CannotDeleteProvider must be restricted to users with 'osf.delete_collectionprovider'.""" + + @pytest.fixture() + def provider(self): + return CollectionProviderFactory() + + def test_denied_for_general_user(self, req, provider): + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, collection_provider_id=provider.id) + + def test_denied_for_anonymous(self, provider): + req = RequestFactory().get('/fake_path') + req.user = AnonymousUser() + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, collection_provider_id=provider.id) + + def test_allowed_for_user_with_permission(self, req, user, provider): + permission = Permission.objects.get(codename='delete_collectionprovider') + user.user_permissions.add(permission) + user.save() + + res = views.CannotDeleteProvider.as_view()(req, collection_provider_id=provider.id) + assert res.status_code == 200 + + def test_allowed_for_superuser(self, req, user, provider): + user.is_superuser = True + user.save() + + res = views.CannotDeleteProvider.as_view()(req, collection_provider_id=provider.id) + assert res.status_code == 200 + + def test_404_for_missing_provider(self, req, user): + permission = Permission.objects.get(codename='delete_collectionprovider') + user.user_permissions.add(permission) + user.save() + + with pytest.raises(Http404): + views.CannotDeleteProvider.as_view()(req, collection_provider_id=999999) diff --git a/admin_tests/common_auth/test_views.py b/admin_tests/common_auth/test_views.py index 5bc23647714..89bc21614f7 100644 --- a/admin_tests/common_auth/test_views.py +++ b/admin_tests/common_auth/test_views.py @@ -5,14 +5,15 @@ from django.http import Http404 from django.urls import reverse from django.contrib.auth import REDIRECT_FIELD_NAME +from django.contrib.messages.storage.fallback import FallbackStorage from tests.base import AdminTestCase from osf_tests.factories import AuthUserFactory, InstitutionFactory from admin_tests.utilities import setup_form_view, setup_view from osf.models.user import OSFUser -from admin.common_auth.views import RegisterUser, ShibLoginView -from admin.common_auth.forms import UserRegistrationForm +from admin.common_auth.views import RegisterUser, ShibLoginView, LoginView +from admin.common_auth.forms import UserRegistrationForm, LoginForm class TestRegisterUser(AdminTestCase): @@ -356,3 +357,47 @@ def test_new_user_missing_displayname_header_raises_key_error(self, mock_use_epp view = setup_view(ShibLoginView(), request) with nt.assert_raises(KeyError): view.dispatch(request) + + +class TestLoginViewFormValid(AdminTestCase): + """LoginView.form_valid must skip authenticate() entirely when + ENABLE_LOGIN_FORM is False, so password-based login stays disabled even via + a direct POST to /account/login/.""" + + def setUp(self): + super(TestLoginViewFormValid, self).setUp() + self.view = LoginView() + self.request = RequestFactory().post('/account/login/') + # django.contrib.messages has a bug which effects unittests + # more info here -> https://code.djangoproject.com/ticket/17971 + setattr(self.request, 'session', 'session') + setattr(self.request, '_messages', FallbackStorage(self.request)) + self.form = LoginForm(data={'email': 'user@example.com', 'password': 'secret'}) + nt.assert_true(self.form.is_valid()) + self.view = setup_form_view(self.view, self.request, self.form) + + @mock.patch('admin.common_auth.views.ENABLE_LOGIN_FORM', False) + @mock.patch('admin.common_auth.views.authenticate') + def test_authenticate_skipped_when_login_form_disabled(self, mock_authenticate): + response = self.view.form_valid(self.form) + nt.assert_false(mock_authenticate.called) + nt.assert_equal(response.status_code, 302) + nt.assert_true(response.url.startswith('/account/login')) + + @mock.patch('admin.common_auth.views.ENABLE_LOGIN_FORM', True) + @mock.patch('admin.common_auth.views.authenticate', return_value=None) + def test_authenticate_called_and_denied_when_login_form_enabled(self, mock_authenticate): + response = self.view.form_valid(self.form) + nt.assert_true(mock_authenticate.called) + nt.assert_equal(response.status_code, 302) + nt.assert_true(response.url.startswith('/account/login')) + + @mock.patch('admin.common_auth.views.ENABLE_LOGIN_FORM', True) + @mock.patch('admin.common_auth.views.login') + @mock.patch('admin.common_auth.views.authenticate') + def test_authenticate_called_and_login_succeeds_when_login_form_enabled(self, mock_authenticate, mock_login): + mock_authenticate.return_value = AuthUserFactory() + response = self.view.form_valid(self.form) + nt.assert_true(mock_authenticate.called) + nt.assert_true(mock_login.called) + nt.assert_equal(response.status_code, 302) diff --git a/admin_tests/management/__init__.py b/admin_tests/management/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/admin_tests/management/test_views.py b/admin_tests/management/test_views.py new file mode 100644 index 00000000000..8e93b66af0f --- /dev/null +++ b/admin_tests/management/test_views.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +from nose import tools as nt +import mock +from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser +from django.core.exceptions import PermissionDenied + +from admin.management import views +from osf_tests.factories import AuthUserFactory +from tests.base import AdminTestCase + + +class TestManagementCommandsPermission(AdminTestCase): + """ManagementCommands must be restricted to Integrated Admin (is_super_admin).""" + + def setUp(self): + super(TestManagementCommandsPermission, self).setUp() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + self.general_user = AuthUserFactory() + + def test_get_denied_for_general_user(self): + request = RequestFactory().get('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.ManagementCommands.as_view()(request) + + def test_get_denied_for_anonymous(self): + request = RequestFactory().get('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.ManagementCommands.as_view()(request) + + def test_get_allowed_for_superuser(self): + request = RequestFactory().get('/fake_path') + request.user = self.superuser + response = views.ManagementCommands.as_view()(request) + nt.assert_equal(response.status_code, 200) + + +class TestWaffleFlagPermission(AdminTestCase): + """WaffleFlag must be restricted to Integrated Admin (is_super_admin).""" + + def setUp(self): + super(TestWaffleFlagPermission, self).setUp() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + self.general_user = AuthUserFactory() + + def test_post_denied_for_general_user(self): + request = RequestFactory().post('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.WaffleFlag.as_view()(request) + + def test_post_denied_for_anonymous(self): + request = RequestFactory().post('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.WaffleFlag.as_view()(request) + + @mock.patch('admin.management.views.manage_waffle') + def test_post_allowed_for_superuser(self, mock_manage_waffle): + request = RequestFactory().post('/fake_path') + request.user = self.superuser + response = views.WaffleFlag.as_view()(request) + nt.assert_equal(response.status_code, 302) + nt.assert_true(mock_manage_waffle.called) diff --git a/admin_tests/preprint_providers/test_views.py b/admin_tests/preprint_providers/test_views.py index 66403b89536..204107770a3 100644 --- a/admin_tests/preprint_providers/test_views.py +++ b/admin_tests/preprint_providers/test_views.py @@ -6,6 +6,9 @@ import responses from nose import tools as nt from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser, Permission +from django.core.exceptions import PermissionDenied +from django.http import Http404 from django.core.files.uploadedfile import InMemoryUploadedFile from scripts.update_taxonomies import update_taxonomies @@ -369,6 +372,48 @@ def test_cannot_get_if_preprints_present(self, req, view, preprint, provider_wit assert redirect.url == '/preprint_providers/{}/cannot_delete/'.format(provider_with_preprint.id) assert redirect.status_code == 302 +@pytest.mark.urls('admin.base.urls') +class TestCannotDeleteProviderPermission: + """CannotDeleteProvider must be restricted to users with 'osf.delete_preprintprovider'.""" + + @pytest.fixture() + def provider(self): + return PreprintProviderFactory() + + def test_denied_for_general_user(self, req, provider): + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id) + + def test_denied_for_anonymous(self, provider): + req = RequestFactory().get('/fake_path') + req.user = AnonymousUser() + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id) + + def test_allowed_for_user_with_permission(self, req, user, provider): + permission = Permission.objects.get(codename='delete_preprintprovider') + user.user_permissions.add(permission) + user.save() + + res = views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id) + assert res.status_code == 200 + + def test_allowed_for_superuser(self, req, user, provider): + user.is_superuser = True + user.save() + + res = views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id) + assert res.status_code == 200 + + def test_404_for_missing_provider(self, req, user): + permission = Permission.objects.get(codename='delete_preprintprovider') + user.user_permissions.add(permission) + user.save() + + with pytest.raises(Http404): + views.CannotDeleteProvider.as_view()(req, preprint_provider_id=999999) + + class TestProcessCustomTaxonomy(ProcessCustomTaxonomyMixinBase): @pytest.fixture(autouse=True) diff --git a/admin_tests/quota_recalc/test_views.py b/admin_tests/quota_recalc/test_views.py index cbaa876e015..1263d179b26 100644 --- a/admin_tests/quota_recalc/test_views.py +++ b/admin_tests/quota_recalc/test_views.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser +from django.core.exceptions import PermissionDenied import json import mock from nose import tools as nt @@ -12,9 +14,16 @@ class TestQuotaRecalcView(AdminTestCase): - @staticmethod - def get_request(view, **kwargs): - return view(RequestFactory().get('/fake_path'), **kwargs) + def setUp(self): + super(TestQuotaRecalcView, self).setUp() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + + def get_request(self, view, **kwargs): + request = RequestFactory().get('/fake_path') + request.user = self.superuser + return view(request, **kwargs) @mock.patch('admin.quota_recalc.views.used_quota') def test_user_create_userquota_record(self, mock_usedquota): @@ -85,7 +94,59 @@ def test_users_create_userquota_record(self, mock_usedquota): res_json3 = json.loads(response3.content) nt.assert_equal(response3.status_code, 200) nt.assert_equal(res_json3['status'], 'OK') - nt.assert_true('2' in res_json3['message']) + nt.assert_true('3' in res_json3['message']) + + +class TestQuotaRecalcPermission(AdminTestCase): + """all_users/user must be restricted to Integrated Admin (is_superuser).""" + + def setUp(self): + super(TestQuotaRecalcPermission, self).setUp() + self.general_user = AuthUserFactory() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + self.target_user = AuthUserFactory() + + def test_all_users_denied_for_general_user(self): + request = RequestFactory().get('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.all_users(request) + + def test_all_users_denied_for_anonymous(self): + request = RequestFactory().get('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.all_users(request) + + @mock.patch('admin.quota_recalc.views.used_quota') + def test_all_users_allowed_for_superuser(self, mock_usedquota): + mock_usedquota.return_value = 0 + request = RequestFactory().get('/fake_path') + request.user = self.superuser + response = views.all_users(request) + nt.assert_equal(response.status_code, 200) + + def test_user_denied_for_general_user(self): + request = RequestFactory().get('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.user(request, guid=self.target_user._id) + + def test_user_denied_for_anonymous(self): + request = RequestFactory().get('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.user(request, guid=self.target_user._id) + + @mock.patch('admin.quota_recalc.views.used_quota') + def test_user_allowed_for_superuser(self, mock_usedquota): + mock_usedquota.return_value = 0 + request = RequestFactory().get('/fake_path') + request.user = self.superuser + response = views.user(request, guid=self.target_user._id) + nt.assert_equal(response.status_code, 200) class TestCalculateQuota(AdminTestCase): diff --git a/admin_tests/rdm_statistics/test_views.py b/admin_tests/rdm_statistics/test_views.py index 670cfcc31d7..97291392f09 100644 --- a/admin_tests/rdm_statistics/test_views.py +++ b/admin_tests/rdm_statistics/test_views.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- from nose import tools as nt from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser +from django.core.exceptions import PermissionDenied from tests.base import AdminTestCase from osf_tests.factories import ( @@ -427,7 +429,7 @@ def test_get(self, *args, **kwargs): nt.assert_equal(len(resp), 3) def test_send_stat_mail(self, *args, **kwargs): - nt.assert_equal(views.send_stat_mail(self.request).status_code, 200) + nt.assert_equal(views.send_stat_mail_core(self.request).status_code, 200) def test_send_error_mail(self, *args, **kwargs): ret = views.send_error_mail(Exception()) @@ -505,3 +507,78 @@ def test_get_graphs(self, mock_sessionget): provider='osfstorage' ) nt.assert_equal(result['content-type'], 'image/png') + + +class TestSendStatMailPermission(AdminTestCase): + """send_stat_mail (No.86, /statistics/test/mail/) must be restricted to Integrated Admin (is_superuser).""" + + def setUp(self): + super(TestSendStatMailPermission, self).setUp() + self.general_user = AuthUserFactory() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + + def test_send_stat_mail_denied_for_general_user(self): + request = RequestFactory().get('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.send_stat_mail(request) + + def test_send_stat_mail_denied_for_anonymous(self): + request = RequestFactory().get('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.send_stat_mail(request) + + def test_send_stat_mail_allowed_for_superuser(self): + request = RequestFactory().get('/fake_path') + request.user = self.superuser + response = views.send_stat_mail(request) + nt.assert_equal(response.status_code, 200) + + +class TestIndexViewPermission(AdminTestCase): + """IndexView must be restricted to Integrated Admin or Institutional Admin.""" + + def setUp(self): + super(TestIndexViewPermission, self).setUp() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + self.institution_admin = AuthUserFactory() + self.institution_admin.is_staff = True + self.institution_admin.save() + self.general_user = AuthUserFactory() + + def test_denied_for_general_user(self): + request = RequestFactory().get('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.IndexView.as_view()(request) + + def test_denied_for_anonymous(self): + request = RequestFactory().get('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.IndexView.as_view()(request) + + def test_allowed_for_super_admin(self): + # Only test_func (permission check) is exercised here, not the full + # get() flow — IndexView.find_bookmark_collection is currently broken + # and fixing it is out of scope for this change. + request = RequestFactory().get('/fake_path') + request.user = self.superuser + view = views.IndexView() + view.request = request + nt.assert_true(view.test_func()) + + def test_allowed_for_institution_admin(self): + # Only test_func (permission check) is exercised here, not the full + # get() flow — IndexView.find_bookmark_collection is currently broken + # and fixing it is out of scope for this change. + request = RequestFactory().get('/fake_path') + request.user = self.institution_admin + view = views.IndexView() + view.request = request + nt.assert_true(view.test_func()) diff --git a/admin_tests/registration_providers/test_views.py b/admin_tests/registration_providers/test_views.py index 9025c46b68c..e2bc66a9f1d 100644 --- a/admin_tests/registration_providers/test_views.py +++ b/admin_tests/registration_providers/test_views.py @@ -2,6 +2,9 @@ import pytest from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser, Permission +from django.core.exceptions import PermissionDenied +from django.http import Http404 from osf_tests.factories import ( AuthUserFactory, @@ -207,3 +210,79 @@ def test_post(self, view, req, schema, provider): res = view.post(req) assert res.status_code == 302 assert provider.schemas.get(id=schema_id) + + +@pytest.mark.urls('admin.base.urls') +class TestChangeSchemaPermission: + """ChangeSchema must be restricted to users with 'osf.change_registrationprovider'.""" + + @pytest.fixture() + def provider(self): + return RegistrationProviderFactory() + + def test_get_denied_for_general_user(self, req, provider): + with pytest.raises(PermissionDenied): + views.ChangeSchema.as_view()(req, registration_provider_id=provider.id) + + def test_get_denied_for_anonymous(self, provider): + req = RequestFactory().get('/fake_path') + req.user = AnonymousUser() + with pytest.raises(PermissionDenied): + views.ChangeSchema.as_view()(req, registration_provider_id=provider.id) + + def test_get_allowed_for_user_with_permission(self, req, user, provider): + permission = Permission.objects.get(codename='change_registrationprovider') + user.user_permissions.add(permission) + user.save() + + res = views.ChangeSchema.as_view()(req, registration_provider_id=provider.id) + assert res.status_code == 200 + + def test_get_allowed_for_superuser(self, req, user, provider): + user.is_superuser = True + user.save() + + res = views.ChangeSchema.as_view()(req, registration_provider_id=provider.id) + assert res.status_code == 200 + + +@pytest.mark.urls('admin.base.urls') +class TestCannotDeleteProviderPermission: + """CannotDeleteProvider must be restricted to users with 'osf.delete_registrationprovider'.""" + + @pytest.fixture() + def provider(self): + return RegistrationProviderFactory() + + def test_denied_for_general_user(self, req, provider): + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id) + + def test_denied_for_anonymous(self, provider): + req = RequestFactory().get('/fake_path') + req.user = AnonymousUser() + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id) + + def test_allowed_for_user_with_permission(self, req, user, provider): + permission = Permission.objects.get(codename='delete_registrationprovider') + user.user_permissions.add(permission) + user.save() + + res = views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id) + assert res.status_code == 200 + + def test_allowed_for_superuser(self, req, user, provider): + user.is_superuser = True + user.save() + + res = views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id) + assert res.status_code == 200 + + def test_404_for_missing_provider(self, req, user): + permission = Permission.objects.get(codename='delete_registrationprovider') + user.user_permissions.add(permission) + user.save() + + with pytest.raises(Http404): + views.CannotDeleteProvider.as_view()(req, registration_provider_id=999999) From 15d1bc9538c34a2acdc4e24369fa72590141c16b Mon Sep 17 00:00:00 2001 From: NishikawaKoharu Date: Mon, 31 Aug 2026 15:27:12 +0900 Subject: [PATCH 10/41] =?UTF-8?q?y-websocket=20JWT=E8=AA=8D=E8=A8=BC?= =?UTF-8?q?=E3=83=88=E3=83=BC=E3=82=AF=E3=83=B3=E7=99=BA=E8=A1=8C=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/wiki/settings/defaults.py | 5 +++ addons/wiki/templates/edit.mako | 1 + addons/wiki/tests/test_wiki.py | 53 +++++++++++++++++++++++++++++++- addons/wiki/utils.py | 25 +++++++++++++++ addons/wiki/views.py | 3 ++ 5 files changed, 86 insertions(+), 1 deletion(-) diff --git a/addons/wiki/settings/defaults.py b/addons/wiki/settings/defaults.py index 9fadefc5f61..bbf8ed0f0de 100644 --- a/addons/wiki/settings/defaults.py +++ b/addons/wiki/settings/defaults.py @@ -8,6 +8,11 @@ SHAREJS_PORT = 7007 SHAREJS_URL = '{}:{}'.format(SHAREJS_HOST, SHAREJS_PORT) +Y_WEBSOCKET_SECRET = os.environ.get('Y_WEBSOCKET_SECRET', '') +Y_WEBSOCKET_JWT_ALGORITHM = 'HS256' +# Token validity for WebSocket connection (seconds). Not enforced after connection is established. +Y_WEBSOCKET_TOKEN_TTL = int(os.environ.get('Y_WEBSOCKET_TOKEN_TTL', 8 * 60 * 60)) + Y_WEBSOCKET_HOST = 'localhost' Y_WEBSOCKET_PORT = 1234 Y_WEBSOCKET_URL = '{}:{}'.format(Y_WEBSOCKET_HOST, Y_WEBSOCKET_PORT) diff --git a/addons/wiki/templates/edit.mako b/addons/wiki/templates/edit.mako index c4fd10d92da..6356b6f15e6 100644 --- a/addons/wiki/templates/edit.mako +++ b/addons/wiki/templates/edit.mako @@ -522,6 +522,7 @@ ${parent.javascript_bottom()} metadata: { registration: true, docId: ${ sharejs_uuid | sjson, n }, + yWebsocketToken: ${ y_websocket_token | sjson, n }, userId: ${user_id | sjson, n }, userName: ${ user_full_name | sjson, n }, userUrl: ${ user_url | sjson, n }, diff --git a/addons/wiki/tests/test_wiki.py b/addons/wiki/tests/test_wiki.py index 6daffd6dead..62f48f36531 100644 --- a/addons/wiki/tests/test_wiki.py +++ b/addons/wiki/tests/test_wiki.py @@ -35,7 +35,7 @@ from addons.wiki.utils import ( get_sharejs_uuid, generate_private_uuid, share_db, delete_share_doc, migrate_uuid, format_wiki_version, serialize_wiki_settings, serialize_wiki_widget, - check_file_object_in_node + check_file_object_in_node, generate_y_websocket_token ) from addons.wiki.views import WIKI_IMPORT_TASK_ALREADY_EXISTS from addons.wiki import tasks @@ -3380,3 +3380,54 @@ def test_project_update_wiki_page_sort(self): self.assertEqual(result_wiki_child_page1, {'parent_id': wiki_page2_id, 'sort_order': 1}) self.assertEqual(result_wiki_child_page2, {'parent_id': wiki_page2_id, 'sort_order': 2}) self.assertEqual(result_wiki_child_page3, {'parent_id': wiki_child_page2_id, 'sort_order': 1}) + +@pytest.mark.enable_bookmark_creation +@mock.patch('addons.wiki.settings.Y_WEBSOCKET_SECRET', 'test-y-websocket-secret') +class TestYWebsocketToken(OsfTestCase): + + def setUp(self): + super(TestYWebsocketToken, self).setUp() + self.user = AuthUserFactory() + self.project = ProjectFactory(is_public=True, creator=self.user) + self.wname = 'foo.bar' + self.wkey = to_mongo_key(self.wname) + + def test_token_generated_for_editor(self): + import jwt + from addons.wiki import settings as wiki_settings + + url = self.project.web_url_for('project_wiki_view', wname=self.wname) + res = self.app.get(url, auth=self.user.auth) + assert_equal(res.status_code, 200) + + body = res.body.decode() + sharejs_uuid = get_sharejs_uuid(self.project, self.wname) + assert_in(sharejs_uuid, body) + + token_match = re.search(r'"yWebsocketToken":\s*"([^"]+)"', body) + assert_true(token_match) + token = token_match.group(1) + assert_true(token) + + payload = jwt.decode( + token, + wiki_settings.Y_WEBSOCKET_SECRET, + algorithms=[wiki_settings.Y_WEBSOCKET_JWT_ALGORITHM], + ) + assert_equal(payload['doc_id'], sharejs_uuid) + + def test_token_not_visible_without_write_permission(self): + WikiPage.objects.create_for_node(self.project, self.wname, 'some content', Auth(self.user)) + + url = self.project.web_url_for('project_wiki_view', wname=self.wname) + res = self.app.get(url) + assert_equal(res.status_code, 200) + + body = res.body.decode() + assert_not_in(get_sharejs_uuid(self.project, self.wname), body) + assert_not_in('"yWebsocketToken":', body) + + def test_generate_y_websocket_token_without_secret(self): + with mock.patch('addons.wiki.settings.Y_WEBSOCKET_SECRET', ''): + assert_equal(generate_y_websocket_token('doc-id'), '') + diff --git a/addons/wiki/utils.py b/addons/wiki/utils.py index 39af0660fb0..51304a4410c 100644 --- a/addons/wiki/utils.py +++ b/addons/wiki/utils.py @@ -4,10 +4,13 @@ import uuid import unicodedata import ssl +import datetime from future.moves.urllib.parse import quote +import jwt from pymongo import MongoClient import requests +from django.utils import timezone from bs4 import BeautifulSoup from django.apps import apps @@ -65,6 +68,28 @@ def get_sharejs_uuid(node, wname): )) if private_uuid else None +def generate_y_websocket_token(doc_id): + """ + Generate a signed JWT for y-websocket connection authorization. + Returns an empty string when Y_WEBSOCKET_SECRET is not configured. + """ + secret = wiki_settings.Y_WEBSOCKET_SECRET + if not secret or not doc_id: + return '' + + payload = { + 'doc_id': doc_id, + 'exp': timezone.now() + datetime.timedelta(seconds=wiki_settings.Y_WEBSOCKET_TOKEN_TTL), + } + token = jwt.encode( + payload, + secret, + algorithm=wiki_settings.Y_WEBSOCKET_JWT_ALGORITHM, + ) + if isinstance(token, bytes): + return token.decode() + return token + def delete_share_doc(node, wname): """Deletes share document and removes namespace from model.""" diff --git a/addons/wiki/views.py b/addons/wiki/views.py index a1051ba9110..952daf12838 100644 --- a/addons/wiki/views.py +++ b/addons/wiki/views.py @@ -325,10 +325,12 @@ def project_wiki_view(auth, wname, path=None, **kwargs): rendered_before_update = False markdown = '' + y_websocket_token = '' if can_edit: if wiki_key not in node.wiki_private_uuids: wiki_utils.generate_private_uuid(node, wiki_name) sharejs_uuid = wiki_utils.get_sharejs_uuid(node, wiki_name) + y_websocket_token = wiki_utils.generate_y_websocket_token(sharejs_uuid) else: if not wiki_page and wiki_key != 'home': raise WIKI_PAGE_NOT_FOUND_ERROR @@ -366,6 +368,7 @@ def project_wiki_view(auth, wname, path=None, **kwargs): 'sharejs_uuid': sharejs_uuid or '', 'sharejs_url': settings.SHAREJS_URL, 'y_websocket_url': settings.Y_WEBSOCKET_URL, + 'y_websocket_token': y_websocket_token, 'is_current': is_current, 'version_settings': version_settings, 'pages_current': _get_wiki_pages_latest(node), From c66bb89827f76763db02a91d9121ff7f59709a7d Mon Sep 17 00:00:00 2001 From: hcphat Date: Thu, 13 Aug 2026 15:06:39 +0700 Subject: [PATCH 11/41] =?UTF-8?q?ref=202.2.=E3=82=A2=E3=83=89=E3=82=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AB?= =?UTF-8?q?=E7=B5=A1=E3=82=80=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E6=99=82=E3=81=AE=E3=82=AF=E3=82=AA=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=AE=A1=E7=90=86:=20Update=20logic=20pre-check=20quota,=20max?= =?UTF-8?q?=20size=20limit=20when=20move/copy=20and=20update=20UT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/base/views.py | 7 +- .../tests/test_views_copy_move_quota.py | 122 +++- addons/osfstorage/views.py | 29 +- tests/test_addons.py | 19 + tests/test_quota.py | 76 ++- website/project/views/quota.py | 9 +- website/static/js/fangorn.js | 231 +++++-- website/static/js/tests/fangorn.test.js | 643 ++++++++++++++++++ website/translations/js_messages.pot | 2 +- 9 files changed, 1063 insertions(+), 75 deletions(-) diff --git a/addons/base/views.py b/addons/base/views.py index eb63448a331..d373dc6815f 100644 --- a/addons/base/views.py +++ b/addons/base/views.py @@ -436,7 +436,12 @@ def get_auth(auth, **kwargs): waterbutler_settings = node.serialize_waterbutler_settings(provider_name) if is_node_process and provider_settings: - waterbutler_settings['max_file_size'] = getattr(provider_settings.config, 'max_file_size', None) + # Same limit the file browser shows; + # Only osfstorage defines `high_max_file_size`. + max_size = getattr(provider_settings.config, 'max_file_size', None) + if auth.user and 'high_upload_limit' in auth.user.system_tags: + max_size = getattr(provider_settings.config, 'high_max_file_size', None) or max_size + waterbutler_settings['max_file_size'] = max_size if not is_node_process: # for only location_id value diff --git a/addons/osfstorage/tests/test_views_copy_move_quota.py b/addons/osfstorage/tests/test_views_copy_move_quota.py index 9e1c7a36e2d..2364b770363 100644 --- a/addons/osfstorage/tests/test_views_copy_move_quota.py +++ b/addons/osfstorage/tests/test_views_copy_move_quota.py @@ -474,7 +474,9 @@ def test_move_file_inter_target_subtracts_source_quota(self, mock_st, mock_uq): Moving a FILE to a *different* target node must call update_quota on the source target with add=False. """ - other_project = ProjectFactory(creator=self.user) + # Different creator: keeps this test on the "different UserQuota record" + # branch of osfstorage_move_hook, distinct from the same-creator tests below. + other_project = ProjectFactory() other_root = other_project.get_addon('osfstorage').get_root() dest_folder = other_root.append_folder('remote') @@ -535,7 +537,9 @@ def test_move_folder_aggregates_fileinfo(self, mock_st, mock_uq): Moving a FOLDER must aggregate FileInfo sizes for all descendants (new_size) and pass that to update_quota on the source (add=False). """ - other_project = ProjectFactory(creator=self.user) + # Different creator: keeps this test on the "different UserQuota record" + # branch of osfstorage_move_hook, distinct from the same-creator tests below. + other_project = ProjectFactory() other_root = other_project.get_addon('osfstorage').get_root() dest_folder = other_root.append_folder('folder_dest') @@ -568,7 +572,9 @@ def test_move_folder_replaced_size_zeroed(self, mock_st, mock_uq): caller passes a non-zero replaced_size for a folder move the effective delta equals new_size, NOT new_size - replaced_size. """ - other_project = ProjectFactory(creator=self.user) + # Different creator: keeps this test on the "different UserQuota record" + # branch of osfstorage_move_hook, distinct from the same-creator tests below. + other_project = ProjectFactory() other_root = other_project.get_addon('osfstorage').get_root() dest_folder = other_root.append_folder('folder_dest2') @@ -587,6 +593,116 @@ def test_move_folder_replaced_size_zeroed(self, mock_st, mock_uq): sizes_added = [c[0][1] for c in add_calls] assert 300 in sizes_added + # --- SAME-CREATOR CROSS-NODE MOVE (branch by UserQuota record, not node) --- + # These exercise osfstorage_move_hook against a REAL UserQuota row (update_quota is + # not mocked) because the bug being fixed is a real arithmetic corruption of `used`, + # not just which mock gets called. + + def test_move_file_to_other_node_same_creator_does_not_clamp_used_to_zero(self): + """A move to a different node owned by the same creator shares one UserQuota row; + `used` must not clamp to 0 between the two opposing writes on that row.""" + from osf.models import UserQuota + + UserQuota.objects.get_or_create( + user=self.user, storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 10000, 'used': 100}, + ) + + other_project = ProjectFactory(creator=self.user) + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('same_creator_dest') + + version = factories.FileVersionFactory(size=500) + src = self.root_node.append_file('same_creator.txt') + src.add_version(version); src.save() + + resp = self._move(src, dest_folder, dest_target=other_project, replaced_size=0) + assert resp.status_code == 200 + + used = UserQuota.objects.get(user=self.user, storage_type=UserQuota.NII_STORAGE).used + assert used == 100 + + def test_move_file_to_other_node_same_creator_with_replace_subtracts_only_replaced(self): + """Same as above, but the destination overwrites a 50-byte item: net delta is -50.""" + from osf.models import UserQuota + + UserQuota.objects.get_or_create( + user=self.user, storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 10000, 'used': 100}, + ) + + other_project = ProjectFactory(creator=self.user) + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('same_creator_dest_replace') + + version = factories.FileVersionFactory(size=500) + src = self.root_node.append_file('same_creator_replace.txt') + src.add_version(version); src.save() + + resp = self._move(src, dest_folder, dest_target=other_project, replaced_size=50) + assert resp.status_code == 200 + + used = UserQuota.objects.get(user=self.user, storage_type=UserQuota.NII_STORAGE).used + assert used == 50 + + def test_move_file_to_other_node_different_creator_still_transfers_full_size(self): + """A move between nodes with different creators still hits two distinct UserQuota + rows and transfers the full size.""" + from osf.models import UserQuota + + UserQuota.objects.get_or_create( + user=self.user, storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 10000, 'used': 1000}, + ) + other_project = ProjectFactory() + UserQuota.objects.get_or_create( + user=other_project.creator, storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 10000, 'used': 0}, + ) + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('different_creator_dest') + + version = factories.FileVersionFactory(size=500) + src = self.root_node.append_file('different_creator.txt') + src.add_version(version); src.save() + + resp = self._move(src, dest_folder, dest_target=other_project, replaced_size=0) + assert resp.status_code == 200 + + src_used = UserQuota.objects.get(user=self.user, storage_type=UserQuota.NII_STORAGE).used + dest_used = UserQuota.objects.get( + user=other_project.creator, storage_type=UserQuota.NII_STORAGE + ).used + assert src_used == 500 + assert dest_used == 500 + + def test_move_folder_to_other_node_same_creator_is_quota_neutral(self): + """A folder move within the same UserQuota record results in no quota change and + no UserQuota.save() call.""" + from osf.models import UserQuota + + UserQuota.objects.get_or_create( + user=self.user, storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 10000, 'used': 100}, + ) + + other_project = ProjectFactory(creator=self.user) + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('same_creator_folder_dest') + + src_folder = self.root_node.append_folder('same_creator_move_folder') + v = factories.FileVersionFactory(size=500) + f = src_folder.append_file('d.txt'); f.add_version(v); f.save() + FileInfo.objects.update_or_create(file=f, defaults={'file_size': 500}) + + with mock.patch.object(UserQuota, 'save') as mock_save: + resp = self._move(src_folder, dest_folder, dest_target=other_project, replaced_size=999) + assert resp.status_code == 200 + mock_save.assert_not_called() + + used = UserQuota.objects.get(user=self.user, storage_type=UserQuota.NII_STORAGE).used + assert used == 100 + # --------------------------------------------------------------------------- # osfstorage_create_child – FileInfo UPSERT + quota delta diff --git a/addons/osfstorage/views.py b/addons/osfstorage/views.py index ef37de8d090..30203cf57e0 100644 --- a/addons/osfstorage/views.py +++ b/addons/osfstorage/views.py @@ -243,16 +243,33 @@ def osfstorage_move_hook(source, destination, name=None, **kwargs): replaced_size = 0 # osfstorage_delete already subtracted this # UserQuota - if source_target != destination.target: - src_storage = get_project_storage_type(source_target) - dest_storage = get_project_storage_type(destination.target) + src_storage = get_project_storage_type(source_target) + dest_storage = get_project_storage_type(destination.target) + + # "Same UserQuota record" = same (creator, storage_type), matching the criterion + # WaterButler's resolve_quota_context() uses for the pre-check (user_guid + + # storage_type). + # Node identity is NOT the right test: a move to another component owned by the + # same creator lands on the very same UserQuota row, so subtracting from the + # source and adding to the destination would be two writes against one row -- + # and update_quota() floors `used` at 0 between them, which permanently corrupts + # the row whenever used < new_size. + same_user_quota = ( + source_target.creator_id == destination.target.creator_id and + src_storage == dest_storage + ) + + if same_user_quota: + # Net delta is -replaced_size: the moved bytes are already counted in this + # very row, only the overwritten bytes are freed. (For folders replaced_size + # was reset to 0 above -- osfstorage_delete already subtracted them.) + if replaced_size > 0: + update_quota(destination.target, replaced_size, dest_storage, add=False) + else: update_quota(source_target, new_size, src_storage, add=False) delta = new_size - replaced_size if delta != 0: update_quota(destination.target, abs(delta), dest_storage, add=(delta > 0)) - elif replaced_size > 0: - storage_type = get_project_storage_type(destination.target) - update_quota(destination.target, replaced_size, storage_type, add=False) # once the move is complete recalculate storage for both targets if it's a inter-target move. if is_check_permission and source_target != destination.target: diff --git a/tests/test_addons.py b/tests/test_addons.py index a3a46180e93..2254a03fa75 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -121,6 +121,25 @@ def test_auth_download(self): observed_url.port = expected_url.port assert_equal(expected_url, observed_url) + def test_auth_max_file_size_honors_high_upload_limit_tag(self): + """A user with the `high_upload_limit` system tag gets the high max_file_size + sent to WaterButler.""" + self.user.add_system_tag('high_upload_limit') + self.user.save() + url = self.build_url() + with mock.patch.object(self.node_addon.config, 'high_max_file_size', 500, create=True): + res = self.app.get(url, auth=self.user.auth) + data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data'] + assert_equal(data['settings']['max_file_size'], 500) + + def test_auth_max_file_size_without_high_upload_limit_tag(self): + """An untagged user keeps the addon's ordinary limit even when a high limit exists.""" + url = self.build_url() + with mock.patch.object(self.node_addon.config, 'high_max_file_size', 500, create=True): + res = self.app.get(url, auth=self.user.auth) + data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data'] + assert_equal(data['settings']['max_file_size'], 100) + def test_auth_render_action_returns_200(self): url = self.build_url(action='render') res = self.app.get(url, auth=self.user.auth) diff --git a/tests/test_quota.py b/tests/test_quota.py index a6d73106cf7..d3e29977ba8 100644 --- a/tests/test_quota.py +++ b/tests/test_quota.py @@ -540,9 +540,9 @@ def test_add_file_negative_size(self): user=self.user, event_type=FileLog.FILE_ADDED, payload={ - 'provider': 'osfstorage', + 'provider': 's3compatinstitutions', 'metadata': { - 'provider': 'osfstorage', + 'provider': 's3compatinstitutions', 'name': 'testfile', 'materialized': '/filename', 'path': self.file._id, @@ -594,6 +594,51 @@ def test_delete_file(self): ) assert_equal(user_quota.used, 4500) + @mock.patch('website.util.quota.check_select_for_update') + def test_delete_file_without_select_for_update(self, mock_check_select_for_update): + # Forces node_removed() into the `else` branch (FileInfo.objects.get + # without select_for_update), instead of relying on whether the test + # happens to run inside an atomic transaction. + mock_check_select_for_update.return_value = False + + UserQuota.objects.create( + user=self.project_creator, + storage_type=UserQuota.NII_STORAGE, + max_quota=api_settings.DEFAULT_MAX_QUOTA, + used=5500 + ) + FileInfo.objects.create(file=self.file, file_size=1000) + + self.file.deleted_on = datetime.datetime.now() + self.file.deleted_by = self.user + self.file.type = 'osf.trashedfile' + self.file.save() + + quota.update_used_quota( + self=None, + target=self.node, + user=self.user, + event_type=FileLog.FILE_REMOVED, + payload={ + 'provider': 'osfstorage', + 'metadata': { + 'provider': 'osfstorage', + 'name': 'testfile', + 'materialized': '/filename', + 'path': self.file._id, + 'kind': 'file', + 'extra': {} + } + } + ) + + mock_check_select_for_update.assert_called() + user_quota = UserQuota.objects.get( + storage_type=UserQuota.NII_STORAGE, + user=self.project_creator + ) + assert_equal(user_quota.used, 4500) + def test_delete_file_custom_storage(self): UserQuota.objects.create( user=self.project_creator, @@ -982,9 +1027,9 @@ def test_edit_file_negative_size(self): user=self.user, event_type=FileLog.FILE_UPDATED, payload={ - 'provider': 'osfstorage', + 'provider': 's3compatinstitutions', 'metadata': { - 'provider': 'osfstorage', + 'provider': 's3compatinstitutions', 'name': 'testfile', 'materialized': '/filename', 'path': self.file._id, @@ -1550,6 +1595,20 @@ def test_default_values(self): assert_equal(response.json['max'], api_settings.DEFAULT_MAX_QUOTA * api_settings.SIZE_UNIT_GB) assert_equal(response.json['used'], 0) + def test_user_guid_and_storage_type_present_and_consistent_with_creator(self): + response = self.app.get( + '{}?payload={payload}&signature={signature}'.format( + self.node.api_url_for('waterbutler_creator_quota'), + **signing.sign_data(signing.default_signer, {}) + ) + ) + assert_equal(response.status_code, 200) + assert_in('user_guid', response.json) + assert_in('storage_type', response.json) + from website.util import quota as quota_util + assert_equal(response.json['user_guid'], self.node.creator._id) + assert_equal(response.json['storage_type'], quota_util.get_project_storage_type(self.node)) + def test_used_half_custom_quota(self): UserQuota.objects.create( storage_type=UserQuota.NII_STORAGE, @@ -1616,6 +1675,15 @@ def test_default_values(self): assert_equal(response.json['max'], api_settings.DEFAULT_MAX_QUOTA * api_settings.SIZE_UNIT_GB) assert_equal(response.json['used'], 0) + def test_user_guid_and_storage_type_present_and_consistent_with_creator(self): + response = self.app.get( + self.node.api_url_for('get_creator_quota'), + auth=self.user.auth + ) + assert_equal(response.status_code, 200) + assert_in('user_guid', response.json) + assert_in('storage_type', response.json) + def test_used_half_custom_quota(self): UserQuota.objects.create( storage_type=UserQuota.NII_STORAGE, diff --git a/website/project/views/quota.py b/website/project/views/quota.py index 88814f268a0..8f78fbd5d83 100644 --- a/website/project/views/quota.py +++ b/website/project/views/quota.py @@ -17,10 +17,11 @@ def get_quota_from_pid(pid): """Auxiliary function for getting the quota from a project ID. Used on requests by waterbutler and the user (from browser).""" node = AbstractNode.load(pid) - max_quota, used_quota = quota.get_quota_info( - node.creator, quota.get_project_storage_type(node) - ) + storage_type = quota.get_project_storage_type(node) + max_quota, used_quota = quota.get_quota_info(node.creator, storage_type) return { 'max': max_quota * api_settings.SIZE_UNIT_GB, - 'used': used_quota + 'used': used_quota, + 'user_guid': node.creator._id, + 'storage_type': storage_type, } diff --git a/website/static/js/fangorn.js b/website/static/js/fangorn.js index 394b0cf8133..923c7b9351e 100644 --- a/website/static/js/fangorn.js +++ b/website/static/js/fangorn.js @@ -625,6 +625,73 @@ function _rejectConflictItem(tb, from, to, notRenameOp, conflict) { } } +/** + * True when `node` itself, or any folder anywhere beneath it, has not been + * lazy-loaded yet — meaning the subtree's total size cannot be known client-side. + * + * The file browser lazy-loads a single level at a time, so an expanded folder can + * still contain sub-folders whose own children have never been fetched. Summing + * only what happens to be loaded would under-count the freed size. + */ +function hasUnloadedFolder(node) { + if (node.kind === 'folder' && !node.load) { + return true; + } + return (node.children || []).some(hasUnloadedFolder); +} + +/** + * Size freed up at the destination by an overwrite, in bytes. + * + * Mirrors WaterButler's get_replaced_size(): only conflict === 'replace' overwrites + * anything ('keep' renames the incoming item; omitted conflict defaults to 'warn' — + * both free 0 bytes), and only a same-kind destination item is replaced (osfstorage + * allows a file and folder to share a name). + * + * Returns null if the destination folder's subtree isn't fully loaded (size unknown) — + * callers must skip the client-side check and defer to the server. + */ +function getReplacedSize(to, from, conflict) { + if (conflict !== 'replace') { + return 0; + } + var existing = (to.children || []).filter(function(child) { + return child.data.name === from.data.name && child.kind === from.kind; + })[0]; + if (!existing) { + return 0; + } + if (from.kind !== 'folder') { + return parseInt(existing.data.size, 10) || 0; + } + if (hasUnloadedFolder(existing)) { + // Part of the subtree is not lazy-loaded yet; total size unknown, let the + // server-side check decide. + return null; + } + return getAllChildren(existing).reduce(function(total, child) { + return child.kind === 'file' ? total + (parseInt(child.data.size, 10) || 0) : total; + }, 0); +} + +// max_file_size skip narrows by provider only (browser can't know region); +// server gives the authoritative 413 if it turns out not to qualify. +function shouldSkipSizeCheck(operation, fromProvider, destProvider) { + return operation.status === 'move' && fromProvider === destProvider; +} + +function isSameUserQuota(srcQuota, destQuota) { + return !!(srcQuota && destQuota && + srcQuota.user_guid !== undefined && srcQuota.user_guid !== null && + srcQuota.user_guid === destQuota.user_guid && + srcQuota.storage_type === destQuota.storage_type); +} + +function quotaCheckExceeds(destQuota, totalSize, replacedSize) { + replacedSize = replacedSize || 0; + return (destQuota.used + totalSize - replacedSize) > destQuota.max; +} + function doItemOp(operation, to, from, rename, conflict) { var tb = this; // dismiss old modal immediately to prevent button mashing @@ -633,6 +700,7 @@ function doItemOp(operation, to, from, rename, conflict) { var filesRemaining; var inConflictsQueue = false; var syncMoves; + var showQuotaUsageAlert = false; var notRenameOp = typeof rename === 'undefined'; if (notRenameOp) { @@ -669,71 +737,105 @@ function doItemOp(operation, to, from, rename, conflict) { } if (operation !== OPERATIONS.RENAME) { - var destMaxSize = to.data && to.data.accept && to.data.accept.maxSize; - if (destMaxSize) { - var maxSizeBytes = destMaxSize * 1000000; - var oversizedFiles = _findOversizedFiles(from, maxSizeBytes); - if (oversizedFiles.length > 0) { - var maxSizeDisplay = $osf.humanFileSize(maxSizeBytes, true); - oversizedFiles.forEach(function(oversized) { - var displaySize = $osf.humanFileSize(oversized.data.size, true); - $osf.growl(sprintf( - gettext('File「%1$s」is too large (%2$s). Max file size is %3$s.'), - oversized.data.name, displaySize, maxSizeDisplay - )); - }); - _rejectConflictItem(tb, from, to, notRenameOp, conflict); - return; - } + var destProvider = to.data.provider; + + // Sum size of all files in the item being moved/copied + var totalMoveSize = 0; + if (from.kind === 'file') { + totalMoveSize = parseInt(from.data.size, 10) || 0; + } else { + getAllChildren(from).forEach(function(child) { + if (child.kind === 'file') { + totalMoveSize += parseInt(child.data.size, 10) || 0; + } + }); } - var destProvider = to.data.provider; - if (destProvider === 'osfstorage') { - // Sum size of all files in the item being moved/copied - var totalMoveSize = 0; - if (from.kind === 'file') { - totalMoveSize = parseInt(from.data.size, 10) || 0; - } else { - getAllChildren(from).forEach(function(child) { - if (child.kind === 'file') { - totalMoveSize += parseInt(child.data.size, 10) || 0; - } - }); + var skipSizeCheck = shouldSkipSizeCheck(operation, from.data.provider, destProvider); + + if (!skipSizeCheck) { + var destMaxSize = to.data && to.data.accept && to.data.accept.maxSize; + if (destMaxSize) { + var maxSizeBytes = destMaxSize * 1000000; + var oversizedFiles = _findOversizedFiles(from, maxSizeBytes); + if (oversizedFiles.length > 0) { + var maxSizeDisplay = $osf.humanFileSize(maxSizeBytes, true); + oversizedFiles.forEach(function(oversized) { + var displaySize = $osf.humanFileSize(oversized.data.size, true); + $osf.growl(sprintf( + gettext('File「%1$s」is too large (%2$s). Max file size is %3$s.'), + oversized.data.name, displaySize, maxSizeDisplay + )); + }); + _rejectConflictItem(tb, from, to, notRenameOp, conflict); + return; + } } + } - if (totalMoveSize > 0) { - var quotaResp = $.ajax({ + // Quota context is resolved only after the size check above has run, mirroring + // the server's run_pre_checks order: an oversized file is rejected with 413 + // before creator_quota is ever fetched. + var destQuota = null; + var srcQuota = null; + var skipQuotaCheck = false; + if (destProvider === 'osfstorage' && totalMoveSize > 0) { + // Sequential async:false calls (kept in sync with doItemOp's control flow) + var destQuotaResp = $.ajax({ + async: false, + method: 'GET', + url: to.data.nodeApiUrl + 'get_creator_quota/' + }); + // Only a MOVE from osfstorage needs src quota — it serves just to detect a + // same-record move. A falsy srcQuota means "adds usage". + var srcQuotaResp = null; + if (operation.status === 'move' && from.data.provider === 'osfstorage') { + // Same node ⇒ same UserQuota record: reuse the response already in hand + // instead of asking the very same endpoint twice. + srcQuotaResp = (from.data.nodeApiUrl === to.data.nodeApiUrl) ? destQuotaResp : $.ajax({ async: false, method: 'GET', - url: to.data.nodeApiUrl + 'get_creator_quota/' + url: from.data.nodeApiUrl + 'get_creator_quota/' }); - if (quotaResp.responseJSON) { - var quotaData = quotaResp.responseJSON; - var quotaMsgText = ''; + } + if (destQuotaResp.responseJSON && (!srcQuotaResp || srcQuotaResp.responseJSON)) { + destQuota = destQuotaResp.responseJSON; + srcQuota = srcQuotaResp ? srcQuotaResp.responseJSON : null; + skipQuotaCheck = (operation.status === 'move' && + isSameUserQuota(srcQuota, destQuota)); + } + // A failed request leaves destQuota null: the browser evaluates nothing here — + // neither the quota check nor the threshold warning runs below — and defers + // entirely to the server-side check. + } - if (quotaData.used + totalMoveSize > quotaData.max) { - if (from.kind === 'file') { - quotaMsgText = sprintf(gettext('Not enough quota to move/copy the file.')); - } else { - var folderSize = $osf.humanFileSize(totalMoveSize, true); - quotaMsgText = sprintf(gettext('Not enough quota to move/copy. The total size of the folder %1$s.'), folderSize); - } - from.notify.update(quotaMsgText, 'warning', undefined, 3000); - _rejectConflictItem(tb, from, to, notRenameOp, conflict); - return; - } - if (quotaData.used + totalMoveSize > quotaData.max * window.contextVars.threshold) { - $osf.growl( - gettext('Quota usage alert'), - sprintf( - gettext('You have used more than %1$s%% of your quota.'), - (window.contextVars.threshold * 100) - ), - 'warning' - ); - } + // destQuota is only set for an osfstorage destination with something to weigh. + // skipQuotaCheck is the single source of truth for same-UserQuota-record moves: + // by this point that case is already excluded, so nothing below needs to re-check it. + if (destQuota && !skipQuotaCheck) { + var replacedSize = getReplacedSize(to, from, conflict); + var quotaMsgText = ''; + + // replacedSize === null: destination folder not yet loaded, so its size + // is unknown — skip only this blocking check and let the move/copy proceed; + // the server-side check is authoritative. + if (replacedSize !== null && quotaCheckExceeds(destQuota, totalMoveSize, replacedSize)) { + if (from.kind === 'file') { + quotaMsgText = sprintf(gettext('Not enough quota to move/copy the file.')); + } else { + var folderSize = $osf.humanFileSize(totalMoveSize, true); + quotaMsgText = sprintf(gettext('Not enough quota to move/copy. The total size of the folder %1$s.'), folderSize); } + from.notify.update(quotaMsgText, 'warning', undefined, 3000); + _rejectConflictItem(tb, from, to, notRenameOp, conflict); + return; } + // Unknown replacedSize is treated as 0 (worst case) so the alert can still fire. + var quotaDelta = totalMoveSize - (replacedSize || 0); + // Shown only after a confirmed success below. The alert announces *crossing* + // the threshold, so an operation that adds no usage must stay silent. + showQuotaUsageAlert = (quotaDelta > 0 && + destQuota.used + quotaDelta > destQuota.max * window.contextVars.threshold); } } @@ -806,6 +908,17 @@ function doItemOp(operation, to, from, rename, conflict) { from.data.status = undefined; from.notify.update(sprintf(gettext('Successfully %1$s.') , gettext(operation.passed)), 'success', null, 1000); + if (showQuotaUsageAlert) { + $osf.growl( + gettext('Quota usage alert'), + sprintf( + gettext('You have used more than %1$s%% of your quota.'), + (window.contextVars.threshold * 100) + ), + 'warning' + ); + } + if (xhr.status === 200) { to.children.forEach(function(child) { if (child.data.name === from.data.name && child.id !== from.id) { @@ -4294,5 +4407,11 @@ module.exports = { getCopyMode : getCopyMode, showDeleteMultiple : showDeleteMultiple, checkConflicts : checkConflicts, - getPersistentLinkFor: getPersistentLinkFor + getPersistentLinkFor: getPersistentLinkFor, + quotaCheckExceeds: quotaCheckExceeds, + getReplacedSize: getReplacedSize, + isSameUserQuota: isSameUserQuota, + shouldSkipSizeCheck: shouldSkipSizeCheck, + doItemOp: doItemOp, + OPERATIONS: OPERATIONS }; diff --git a/website/static/js/tests/fangorn.test.js b/website/static/js/tests/fangorn.test.js index 43afe7bc7b0..a7951cf99e5 100644 --- a/website/static/js/tests/fangorn.test.js +++ b/website/static/js/tests/fangorn.test.js @@ -13,6 +13,8 @@ var Raven = require('raven-js'); window.contextVars = { osfSupportEmail : 'fake-support@osf.io', + waterbutlerURL : 'https://files.osf.io/', + threshold : 0.9, }; var language = require('js/osfLanguage').projectSettings; @@ -422,5 +424,646 @@ describe('fangorn', () => { assert.equal(Fangorn.showDeleteMultiple([folder, item]), true); }); }); + + describe('shouldSkipSizeCheck', () => { + it('returns true for a move where source and destination provider match', () => { + var operation = {status: 'move'}; + assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 'osfstorage'), true); + }); + + it('returns false for a move where source and destination provider differ', () => { + var operation = {status: 'move'}; + assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 's3'), false); + }); + + it('returns false for a copy even when provider matches', () => { + var operation = {status: 'copy'}; + assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 'osfstorage'), false); + }); + + it('returns false for a copy where provider also differs', () => { + var operation = {status: 'copy'}; + assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 's3'), false); + }); + }); + + describe('isSameUserQuota', () => { + it('returns true when user_guid and storage_type both match', () => { + var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + var dest = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, dest), true); + }); + + it('returns false when user_guid differs', () => { + var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + var dest = {user_guid: 'xyz', storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, dest), false); + }); + + it('returns false when storage_type differs', () => { + var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + var dest = {user_guid: 'abc', storage_type: 'CUSTOM_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, dest), false); + }); + + it('returns false when user_guid is undefined on either side', () => { + var src = {storage_type: 'NII_STORAGE'}; + var dest = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, dest), false); + }); + + it('returns false when user_guid is null on both sides', () => { + var src = {user_guid: null, storage_type: 'NII_STORAGE'}; + var dest = {user_guid: null, storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, dest), false); + }); + + it('returns false when srcQuota is falsy', () => { + var dest = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(null, dest), false); + }); + + it('returns false when destQuota is falsy', () => { + var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, null), false); + }); + }); + + describe('quotaCheckExceeds', () => { + it('returns true when used + totalSize exceeds max', () => { + var destQuota = {used: 800, max: 1000}; + assert.equal(Fangorn.quotaCheckExceeds(destQuota, 300, 0), true); + }); + + it('returns false when used + totalSize stays within max', () => { + var destQuota = {used: 800, max: 1000}; + assert.equal(Fangorn.quotaCheckExceeds(destQuota, 100, 0), false); + }); + + it('returns false when used + totalSize lands exactly on max', () => { + var destQuota = {used: 800, max: 1000}; + assert.equal(Fangorn.quotaCheckExceeds(destQuota, 200, 0), false); + }); + + it('subtracts replacedSize before comparing to max', () => { + var destQuota = {used: 800, max: 1000}; + // Without replacedSize this would exceed (800 + 300 > 1000), + // but the 150 freed by the overwrite brings it back within max. + assert.equal(Fangorn.quotaCheckExceeds(destQuota, 300, 150), false); + }); + + it('treats a missing replacedSize as 0', () => { + var destQuota = {used: 800, max: 1000}; + assert.equal(Fangorn.quotaCheckExceeds(destQuota, 300), true); + }); + }); + + describe('getReplacedSize', () => { + // Fixture builder for treebeard-like nodes: getReplacedSize/hasUnloadedFolder + // read top-level `kind`/`load`, unlike the `getItem` helper above which only + // sets `data.kind`. + var makeNode = function(kind, name, opts) { + opts = opts || {}; + return { + kind: kind, + data: { + name: name, + size: opts.size + }, + children: opts.children || [], + load: opts.load !== undefined ? opts.load : true + }; + }; + + it('returns 0 when conflict is not replace', () => { + var to = {children: []}; + var from = makeNode('file', 'a.txt', {size: 10}); + assert.equal(Fangorn.getReplacedSize(to, from, 'keep'), 0); + }); + + it('returns 0 when no existing item matches name and kind', () => { + var to = {children: [makeNode('file', 'other.txt', {size: 10})]}; + var from = makeNode('file', 'a.txt', {size: 10}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 0); + }); + + it('does not match when name is the same but kind differs', () => { + var to = {children: [makeNode('folder', 'a', {})]}; + var from = makeNode('file', 'a', {size: 10}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 0); + }); + + it('returns the existing file size when replacing a file', () => { + var existing = makeNode('file', 'a.txt', {size: 500}); + var to = {children: [existing]}; + var from = makeNode('file', 'a.txt', {size: 10}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 500); + }); + + it('returns 0 when the existing file has no size', () => { + var existing = makeNode('file', 'a.txt', {}); + var to = {children: [existing]}; + var from = makeNode('file', 'a.txt', {size: 10}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 0); + }); + + it('sums the sizes of all files in a fully loaded existing folder', () => { + var child1 = makeNode('file', 'x.txt', {size: 100}); + var child2 = makeNode('file', 'y.txt', {size: 200}); + var subfolder = makeNode('folder', 'sub', {children: [child1, child2], load: true}); + var existing = makeNode('folder', 'a', {children: [subfolder], load: true}); + var to = {children: [existing]}; + var from = makeNode('folder', 'a', {}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 300); + }); + + it('returns null when the existing folder itself has not been lazy-loaded', () => { + var existing = makeNode('folder', 'a', {load: false}); + var to = {children: [existing]}; + var from = makeNode('folder', 'a', {}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), null); + }); + + it('returns null when a nested subfolder has not been lazy-loaded', () => { + var unloadedSub = makeNode('folder', 'sub', {load: false}); + var existing = makeNode('folder', 'a', {children: [unloadedSub], load: true}); + var to = {children: [existing]}; + var from = makeNode('folder', 'a', {}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), null); + }); + }); + }); + + describe('doItemOp', () => { + var ajaxStub; + var growlStub; + var ravenStub; + + beforeEach(() => { + growlStub = sinon.stub($osf, 'growl'); + ravenStub = sinon.stub(Raven, 'captureMessage'); + }); + + afterEach(() => { + if (ajaxStub) { ajaxStub.restore(); } + growlStub.restore(); + ravenStub.restore(); + }); + + // Fixture builder for a fake Treebeard node with just enough of the + // real API (move/parent/notify/add/removeSelf) for doItemOp to run. + function makeParent() { + return { + data: { + nodeUrl: 'https://osf.io/proj1/', + nodeApiUrl: 'https://osf.io/proj1/api/', + nodeId: 'proj1', + permissions: {edit: true}, + provider: 'osfstorage' + }, + sortChildren: sinon.stub() + }; + } + + function makeItem(kind, data, id, parentID) { + return { + kind: kind, + data: $.extend({provider: 'osfstorage'}, data), + children: [], + id: id, + parentID: parentID, + inProgress: false, + notify: {update: sinon.stub()}, + move: sinon.stub(), + add: sinon.stub(), + removeSelf: sinon.stub(), + parent: sinon.stub().returns(makeParent()) + }; + } + + // Fixture builder for a fake Treebeard.controller (`this` inside doItemOp). + function makeTb() { + var tb = { + modal: {dismiss: sinon.stub(), update: sinon.stub()}, + pendingFileOps: [], + pendingReadyFiles: [], + moveStates: [], + uploadStates: [], + syncFileMoveCache: {osfstorage: {conflicts: [], ready: []}, s3compat: {conflicts: [], ready: []}}, + clearMultiselect: sinon.stub(), + redraw: sinon.stub(), + select: sinon.stub().returns({hide: sinon.stub(), css: sinon.stub()}), + options: { + lazyLoadPreprocess: function(resp) { return {data: resp}; } + } + }; + tb.createItem = sinon.stub().callsFake(function(data) { + return makeItem(data.kind, data); + }); + tb.buildTree = sinon.stub().callsFake(function(data) { + return makeItem(data.kind, data); + }); + return tb; + } + + // A minimal fake jQuery Deferred supporting the .done()/.fail()/.always() + // chain doItemOp attaches to the final POST call. + function fakeDeferred(config) { + var obj = { + done: function(cb) { + if (config.status === 'done') { cb.apply(null, config.args || []); } + return obj; + }, + fail: function(cb) { + if (config.status === 'fail') { cb.apply(null, config.args || []); } + return obj; + }, + always: function(cb) { cb(); return obj; } + }; + return obj; + } + + function stubAjax(responder) { + ajaxStub = sinon.stub($, 'ajax').callsFake(responder); + } + + function growlCallWithType(type) { + return growlStub.getCalls().some(function(call) { return call.args[2] === type; }); + } + + it('returns without calling the API when dropped back onto its own parent', () => { + var tb = makeTb(); + var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/'}, 5, undefined); + var from = makeItem('file', {name: 'a.txt', size: 10}, 3, 5); + ajaxStub = sinon.stub($, 'ajax'); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(ajaxStub.called, false); + }); + + it('rejects the move without calling the API when a file exceeds the destination max size', () => { + var tb = makeTb(); + // Different providers: shouldSkipSizeCheck only shortcuts a same-provider move, + // so this keeps the client-side oversized check from being bypassed. + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1); + var from = makeItem('file', {name: 'big.txt', size: 5000000}, 3, 1); + ajaxStub = sinon.stub($, 'ajax'); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(ajaxStub.called, false); + assert.equal(growlStub.called, true); + assert.equal(from.inProgress, false); + }); + + it('rejects the copy without calling the API when it would exceed quota', () => { + var tb = makeTb(); + var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/'}, 5, 1); + var from = makeItem('file', {name: 'a.txt', size: 100}, 3, 1); + stubAjax(function(opts) { + if (opts.method === 'GET') { + return {responseJSON: {used: 950, max: 1000, user_guid: 'u1', storage_type: 'NII_STORAGE'}}; + } + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined); + assert.equal(ajaxStub.callCount, 1); + assert.equal(from.notify.update.called, true); + assert.equal(from.inProgress, false); + }); + + it('skips the quota check and proceeds when source and destination share the same UserQuota record', () => { + var tb = makeTb(); + var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1); + var from = makeItem('file', {nodeApiUrl: 'https://osf.io/from/', name: 'a.txt', size: 999999, path: '/a.txt'}, 3, 1); + var sameQuota = {used: 10, max: 1000000000, user_guid: 'u1', storage_type: 'NII_STORAGE'}; + stubAjax(function(opts) { + if (opts.method === 'GET') { + return {responseJSON: sameQuota}; + } + if (opts.type === 'POST') { + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 'osfstorage', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'}, + 'success', + {status: 200} + ]}); + } + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + // dest quota GET + src quota GET (different node) + the real POST + assert.equal(ajaxStub.callCount, 3); + assert.equal(tb.uploadStates.length, 1); + assert.equal(tb.uploadStates[0].success, true); + }); + + it('shows a quota usage alert on success once the destination crosses the warning threshold', () => { + var tb = makeTb(); + var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1); + var from = makeItem('file', {name: 'a.txt', size: 150}, 3, 1); + stubAjax(function(opts) { + if (opts.method === 'GET') { + // used(800) + totalSize(150) = 950 <= max(1000) -> not rejected, + // but 950 > max * threshold(0.9) = 900 -> alert should fire. + return {responseJSON: {used: 800, max: 1000, user_guid: 'u1', storage_type: 'NII_STORAGE'}}; + } + if (opts.type === 'POST') { + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 'osfstorage', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'}, + 'success', + {status: 200} + ]}); + } + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined); + assert.equal(growlCallWithType('warning'), true); + }); + + it('creates a new item via tb.createItem for a COPY and posts a copy action', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + var postedData; + stubAjax(function(opts) { + postedData = JSON.parse(opts.data); + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined); + assert.equal(tb.createItem.called, true); + assert.equal(postedData.action, 'copy'); + }); + + it('posts a rename action for a RENAME operation', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'old.txt'}, 3, 9); + var postedData; + stubAjax(function(opts) { + postedData = JSON.parse(opts.data); + return fakeDeferred({status: 'done', args: [ + {name: 'new.txt', provider: 's3compat', path: '/new.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.RENAME, to, from, 'new.txt', undefined); + assert.equal(postedData.action, 'rename'); + assert.equal(postedData.rename, 'new.txt'); + }); + + it('shows a pending growl and stops when the API responds 202', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'done', args: [{}, 'success', {status: 202}]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(growlCallWithType('info'), true); + assert.equal(tb.uploadStates.length, 0); + }); + + it('shows an oversized-file growl and records a failure when the API rejects with 413', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [ + {status: 413, responseJSON: {oversized_files: [{name: 'big.txt', size: 5000000}], max_size: 1000000}}, + 'error' + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(growlStub.called, true); + assert.equal(tb.uploadStates.length, 1); + assert.equal(tb.uploadStates[0].success, false); + }); + + it('shows the mapped MESSAGE_MAP text when the API rejects with 406 and a known message_key', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [ + {status: 406, responseJSON: {message: 'raw message', message_key: 'quota_exceeded'}}, + 'error' + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(growlStub.called, true); + assert.equal(tb.uploadStates[0].success, false); + }); + + it('reports to Raven and shows a fallback message when the API fails with a generic 500', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [{status: 500}, 'error']}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(ravenStub.called, true); + assert.equal(growlStub.called, true); + }); + + // The cases below close the gaps found in the coverage report + // (RDM-osf.io/coverage/html/js/fangorn.js.html) for _findOversizedFiles + // and doItemOp. See investigation/2026-09-03-fangorn-coverage-findoversizedfiles-doitemop.md. + + it('rejects a folder move and reports oversized files found inside it', () => { + var tb = makeTb(); + // Different providers: shouldSkipSizeCheck only shortcuts a same-provider move. + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1); + var from = makeItem('folder', {name: 'myFolder'}, 3, 1); + var smallChild = makeItem('file', {name: 'small.txt', size: 100}, 10, 3); + var bigChild = makeItem('file', {name: 'big.txt', size: 5000000}, 11, 3); + from.children = [smallChild, bigChild]; + ajaxStub = sinon.stub($, 'ajax'); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(ajaxStub.called, false); + assert.equal(growlStub.called, true); + assert.equal(from.inProgress, false); + }); + + it('sums child file sizes when moving a folder and rejects if quota would be exceeded', () => { + var tb = makeTb(); + var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/'}, 5, 1); + var from = makeItem('folder', {name: 'myFolder'}, 3, 1); + var child = makeItem('file', {name: 'a.txt', size: 200}, 10, 3); + from.children = [child]; + stubAjax(function(opts) { + if (opts.method === 'GET') { + return {responseJSON: {used: 900, max: 1000, user_guid: 'u1', storage_type: 'NII_STORAGE'}}; + } + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined); + assert.equal(ajaxStub.callCount, 1); + assert.equal(from.notify.update.called, true); + assert.equal(from.inProgress, false); + }); + + // _rejectConflictItem isn't exported, so these two exercise it through its only + // real entry points (the oversized-file and quota pre-flight checks in doItemOp) + // to cover the object-identity pendingReadyFiles filter and its doSyncMove advance. + it('keeps other pending files queued when a pre-flight reject leaves the sync queue non-empty', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1); + var from = makeItem('file', {name: 'big.txt', size: 5000000}, 3, 1); + var other = makeItem('file', {name: 'other.txt'}, 77, 1); + tb.pendingReadyFiles = [from, other]; + ajaxStub = sinon.stub($, 'ajax'); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, 'replace'); + // Filtered by object identity, not id: "other" survives untouched. + assert.deepEqual(tb.pendingReadyFiles, [other]); + assert.equal(ajaxStub.called, false); + // Queue still has "other" pending, so doSyncMove must not have run yet. + assert.equal(tb.modal.dismiss.callCount, 1); + assert.equal(tb.modal.update.called, false); + }); + + it('advances the sync move queue via doSyncMove once a pre-flight reject empties pendingReadyFiles', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1); + var from = makeItem('file', {name: 'big.txt', size: 5000000}, 3, 1); + tb.pendingReadyFiles = [from]; + ajaxStub = sinon.stub($, 'ajax'); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, 'replace'); + assert.equal(tb.pendingReadyFiles.length, 0); + // Emptying the queue must trigger doSyncMove -> displayMoveStats, which shows + // the move-status modal because addFileStatus already recorded the rejection + // (displayMoveStats resets tb.moveStates back to [] once it builds the modal). + assert.equal(tb.modal.update.called, true); + }); + + it('shows a "conflicts left to resolve" modal when the sync queue has pending conflicts', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + tb.syncFileMoveCache.s3compat = {conflicts: [{item: from, folder: to}], ready: []}; + stubAjax(function() { + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(tb.modal.update.called, true); + }); + + it('preserves the branch and checks the sync-ready queue when moving a github item', () => { + var tb = makeTb(); + // github isn't in the default syncFileMoveCache fixture; add it so the + // notRenameOp conflicts-check doesn't throw on an undefined entry. + tb.syncFileMoveCache.github = {conflicts: [], ready: []}; + var to = makeItem('folder', {provider: 'github', path: '/'}, 5, 1); + var from = makeItem('file', {provider: 'github', name: 'a.txt', branch: 'main'}, 3, 1); + var postedData; + stubAjax(function(opts) { + postedData = JSON.parse(opts.data); + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 'github', path: '/a.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(postedData.branch, 'main'); + }); + + it('removes the existing item at destination when the server responds 200 with a name conflict', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1); + var existingChild = makeItem('file', {name: 'a.txt', provider: 's3compat'}, 99, 5); + to.children = [existingChild]; + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(existingChild.removeSelf.called, true); + }); + + it('rebuilds the child subtree when the moved item is a folder with children in the response', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1); + var from = makeItem('folder', {provider: 's3compat', name: 'myFolder'}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'done', args: [ + { + name: 'myFolder', provider: 's3compat', path: '/dest/myFolder/', kind: 'folder', + nodeUrl: 'https://osf.io/to/', + children: [{name: 'child.txt', kind: 'file', provider: 's3compat'}] + }, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(tb.buildTree.called, true); + assert.equal(from.add.called, true); + assert.equal(from.open, true); + assert.equal(from.load, true); + }); + + it('removes the newly created item when a COPY operation fails', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [{status: 500}, 'error']}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined); + assert.equal(tb.createItem.called, true); + var createdItem = tb.createItem.returnValues[0]; + assert.equal(createdItem.removeSelf.called, true); + }); + + it('shows the responseJSON message for a non-413/406/500/503 failure', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [ + {status: 400, responseJSON: {message: 'custom failure message'}}, + 'error' + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + var calledWithMessage = growlStub.getCalls().some(function(call) { return call.args[1] === 'custom failure message'; }); + assert.equal(calledWithMessage, true); + }); + + it('uses textStatus as the error message on a 503 response', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [{status: 503}, 'Service Unavailable']}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + var calledWithMessage = growlStub.getCalls().some(function(call) { return call.args[1] === 'Service Unavailable'; }); + assert.equal(calledWithMessage, true); + }); + + it('removes the settled item from pendingReadyFiles once the operation completes', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + var other = makeItem('file', {name: 'other.txt'}, 77, 1); + tb.pendingReadyFiles = [from, other]; + stubAjax(function() { + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.deepEqual(tb.pendingReadyFiles, [other]); + }); }); }); diff --git a/website/translations/js_messages.pot b/website/translations/js_messages.pot index ecaba5d4cda..c3c2fbb08dd 100644 --- a/website/translations/js_messages.pot +++ b/website/translations/js_messages.pot @@ -9242,7 +9242,7 @@ msgstr "" msgid "The Integrated Admin added ${contributors} as contributor(s) to ${node}" msgstr "" -msgid "File「%1$s」 is too large (%2$s). Max file size is %3$s." +msgid "File「%1$s」is too large (%2$s). Max file size is %3$s." msgstr "" msgid "You do not have enough available quota." From 5415e04c0064835fd6f064b0cab16402c3016166 Mon Sep 17 00:00:00 2001 From: NishikawaKoharu Date: Fri, 11 Sep 2026 09:50:32 +0900 Subject: [PATCH 12/41] =?UTF-8?q?Wiki=E3=82=A4=E3=83=B3=E3=83=9D=E3=83=BC?= =?UTF-8?q?=E3=83=88=E6=A4=9C=E8=A8=BC=E3=83=BB=E3=82=BF=E3=82=B9=E3=82=AF?= =?UTF-8?q?=E7=B5=90=E6=9E=9CAPI=E3=81=ABADMIN=E6=A8=A9=E9=99=90=E3=83=81?= =?UTF-8?q?=E3=82=A7=E3=83=83=E3=82=AF=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/wiki/tests/test_wiki.py | 4 ++-- addons/wiki/tests/test_wiki_import.py | 2 +- addons/wiki/views.py | 5 +++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/addons/wiki/tests/test_wiki.py b/addons/wiki/tests/test_wiki.py index 6daffd6dead..a7d37cb5529 100644 --- a/addons/wiki/tests/test_wiki.py +++ b/addons/wiki/tests/test_wiki.py @@ -1861,7 +1861,7 @@ def test_project_wiki_validate_for_import(self, mock_check_file_object_in_node): mock_check_file_object_in_node.return_value = True dir_id = self.root_import_folder1._id url = self.project.api_url_for('project_wiki_validate_for_import', dir_id=dir_id) - res = self.app.get(url) + res = self.app.get(url, auth=self.user.auth) response_json = res.json task_id = response_json['taskId'] uuid_obj = uuid.UUID(task_id) @@ -1875,7 +1875,7 @@ def test_project_wiki_validate_for_import_error(self, mock_check_file_object_in_ )) dir_id = self.root_import_folder1._id url = self.project.api_url_for('project_wiki_validate_for_import', dir_id=dir_id) - res = self.app.get(url, expect_errors=True) + res = self.app.get(url, auth=self.user.auth, expect_errors=True) self.assertEqual(res.status_code, 400) self.assertEqual(res.json['message_short'], 'directory id does not exist') self.assertEqual(res.json['message_long'], 'directory id does not exist') diff --git a/addons/wiki/tests/test_wiki_import.py b/addons/wiki/tests/test_wiki_import.py index 73fe2c3a941..f17fc8dc6d3 100644 --- a/addons/wiki/tests/test_wiki_import.py +++ b/addons/wiki/tests/test_wiki_import.py @@ -1245,7 +1245,7 @@ def test_project_wiki_validate_for_import(self, mock_check_file_object_in_node): mock_check_file_object_in_node.return_value = True dir_id = self.root_import_folder1._id url = self.project.api_url_for('project_wiki_validate_for_import', dir_id=dir_id) - res = self.app.get(url) + res = self.app.get(url, auth=self.user.auth) response_json = res.json task_id = response_json['taskId'] uuid_obj = uuid.UUID(task_id) diff --git a/addons/wiki/views.py b/addons/wiki/views.py index a1051ba9110..afebea3cd01 100644 --- a/addons/wiki/views.py +++ b/addons/wiki/views.py @@ -772,6 +772,9 @@ def serialize_component_wiki(node, auth): return None @must_be_valid_project +@must_have_permission(ADMIN) +@must_not_be_registration +@must_have_addon('wiki', 'node') def project_wiki_validate_for_import(dir_id, node, **kwargs): wiki_utils.check_file_object_in_node(dir_id, node) node_id = node.guids.first()._id @@ -1306,6 +1309,8 @@ def _create_import_error_list(wiki_infos, imported_list): return import_errors @must_be_valid_project +@must_have_permission(ADMIN) +@must_not_be_registration @must_have_addon('wiki', 'node') def project_get_task_result(task_id, node, **kwargs): res = AsyncResult(task_id, app=celery_app) From 14943c8734ac894466101686ef672bc08a89383e Mon Sep 17 00:00:00 2001 From: ndnhat Date: Fri, 11 Sep 2026 16:18:20 +0700 Subject: [PATCH 13/41] =?UTF-8?q?Ref:=20=E6=A9=9F=E9=96=A2=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E8=80=85=E6=A9=9F=E8=83=BDAPI=E3=81=AE=E9=81=A9?= =?UTF-8?q?=E5=88=87=E3=81=AA=E6=A8=A9=E9=99=90=E4=BB=98=E4=B8=8E:=20Updat?= =?UTF-8?q?e=20token=20verify=20for=20the=20following=20endpoint=20/custom?= =?UTF-8?q?=5Fstorage=5Flocation/external=5Facc=5Fupdate//?= =?UTF-8?q?=20/statistics/gather//?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- admin/rdm_custom_storage_location/cron.sh | 2 +- admin/rdm_custom_storage_location/urls.py | 2 +- admin/rdm_custom_storage_location/views.py | 7 +-- admin/rdm_statistics/cron.sh | 2 +- admin/rdm_statistics/urls.py | 2 +- admin/rdm_statistics/views.py | 18 ++---- admin_tests/rdm_statistics/test_views.py | 14 +++-- framework/auth/cron_signed_url.py | 62 +++++++++++++++++++ .../commands/generate_signed_cron_token.py | 19 ++++++ .../test_generate_signed_cron_token.py | 25 ++++++++ tests/test_cron_signed_url.py | 62 +++++++++++++++++++ tests/test_oauth.py | 10 +-- website/settings/__init__.py | 2 +- website/settings/defaults.py | 7 +++ 14 files changed, 203 insertions(+), 31 deletions(-) create mode 100644 framework/auth/cron_signed_url.py create mode 100644 osf/management/commands/generate_signed_cron_token.py create mode 100644 osf_tests/management_commands/test_generate_signed_cron_token.py create mode 100644 tests/test_cron_signed_url.py diff --git a/admin/rdm_custom_storage_location/cron.sh b/admin/rdm_custom_storage_location/cron.sh index 10cb90c4250..81c65448528 100644 --- a/admin/rdm_custom_storage_location/cron.sh +++ b/admin/rdm_custom_storage_location/cron.sh @@ -1 +1 @@ -*/30 * * * * curl http://localhost:8001/custom_storage_location/external_acc_update/d610ef95f0b0f5868f13919b8ed64070b9acb9c19b8da9f2c514ed938203ec3e236c9cad4f4146bdf22b4e79cf0d92f6d4f4c996d236c6b0ee79a1336b26afb7/ | jq -c . +*/30 * * * * TOKEN=$(cd /code && python3 manage.py generate_signed_cron_token | tail -n1) && [ -n "$TOKEN" ] && curl "http://localhost:8001/custom_storage_location/external_acc_update/$TOKEN/" | jq -c . diff --git a/admin/rdm_custom_storage_location/urls.py b/admin/rdm_custom_storage_location/urls.py index 6a62d690294..c51f34c148e 100644 --- a/admin/rdm_custom_storage_location/urls.py +++ b/admin/rdm_custom_storage_location/urls.py @@ -3,7 +3,7 @@ urlpatterns = [ - url(r'^external_acc_update/(?P-?\w+)/$', views.external_acc_update, name='external_acc_update'), + url(r'^external_acc_update/(?P\d+)/(?P[0-9a-fA-F]+)/$', views.external_acc_update, name='external_acc_update'), url(r'^institutional_storage/$', views.InstitutionalStorageView.as_view(), name='institutional_storage'), url(r'^institutional_storage/institutions/$', views.InstitutionalStorageListView.as_view(), name='institutional_storage_institutions'), url(r'^institutional_storage/institutions/(?P[0-9]+)/$', views.InstitutionalStorageView.as_view(), name='institutional_storage_list'), diff --git a/admin/rdm_custom_storage_location/views.py b/admin/rdm_custom_storage_location/views.py index dff08d2b3ec..33dd5824844 100644 --- a/admin/rdm_custom_storage_location/views.py +++ b/admin/rdm_custom_storage_location/views.py @@ -5,7 +5,6 @@ from django.views.generic import TemplateView, View, ListView import codecs import json -import hashlib from rest_framework import status as http_status from mimetypes import MimeTypes import os @@ -19,13 +18,13 @@ from osf.models import Institution, OSFUser from osf.models.external import ExternalAccountTemporary from scripts import refresh_addon_tokens +from framework.auth import cron_signed_url from website import settings as osf_settings from distutils.util import strtobool from admin.base import settings logger = logging.getLogger(__name__) -SITE_KEY = 'rdm_custom_storage_location' INSTITUTION_NOT_FOUND_MESSAGE = 'Institution does not exist' class InstitutionalStorageBaseView(RdmPermissionMixin, UserPassesTestMixin): @@ -522,8 +521,8 @@ def post(self, request, *args, **kwargs): 'message': 'Garbage data removed!!' }, status=http_status.HTTP_200_OK) -def external_acc_update(request, access_token): - if hashlib.sha512(SITE_KEY.encode('utf-8')).hexdigest() != access_token.lower(): +def external_acc_update(request, ts, signature): + if not cron_signed_url.verify_signed_params(ts, signature): return HttpResponse( json.dumps({'state': 'fail', 'error': 'access forbidden'}), content_type='application/json', diff --git a/admin/rdm_statistics/cron.sh b/admin/rdm_statistics/cron.sh index 62a80709268..a3d434e7808 100644 --- a/admin/rdm_statistics/cron.sh +++ b/admin/rdm_statistics/cron.sh @@ -1 +1 @@ -0 2 * * 1 curl http://localhost:8001/statistics/gather/2A85563B2B0F7D3168199F475365F57DA1D56E4BB2CE2B7044EB058AE5E287637E7C636A772682D92C8D6B1830B9A97C5A5DC3DE7016C60BDE4BAA7CC3B38AEB/ | jq -c . +0 2 * * 1 TOKEN=$(cd /code && python3 manage.py generate_signed_cron_token | tail -n1) && [ -n "$TOKEN" ] && curl "http://localhost:8001/statistics/gather/$TOKEN/" | jq -c . diff --git a/admin/rdm_statistics/urls.py b/admin/rdm_statistics/urls.py index 5775b6bf71e..443de572baf 100644 --- a/admin/rdm_statistics/urls.py +++ b/admin/rdm_statistics/urls.py @@ -8,7 +8,7 @@ url(r'^index$', views.IndexView.as_view(), name='index'), url(r'^(?P-?[0-9]+)/graph/(?P\w+)_(?P\w+)\.(\w+)$', views.ImageView.as_view(), name='graph'), - url(r'^gather/(?P-?\w+)/$', views.GatherView.as_view(), name='gather'), + url(r'^gather/(?P\d+)/(?P[0-9a-fA-F]+)/$', views.GatherView.as_view(), name='gather'), url(r'^report/(?P-?[0-9]+)/$', views.create_pdf, name='report'), url(r'^csv/(?P-?[0-9]+)/$', views.create_csv, name='csv'), url(r'^mail/(?P-?[0-9]+)/$', views.SendView.as_view(), name='mail'), diff --git a/admin/rdm_statistics/views.py b/admin/rdm_statistics/views.py index 262c8edce42..9c8f89c2995 100644 --- a/admin/rdm_statistics/views.py +++ b/admin/rdm_statistics/views.py @@ -12,7 +12,6 @@ import csv import pandas as pd import numpy as np -import hashlib from django.apps import apps from django.views.generic import TemplateView, View @@ -44,6 +43,7 @@ from admin.base.utils import superuser_required from admin.rdm.utils import RdmPermissionMixin, get_dummy_institution from admin.rdm_addons import utils +from framework.auth import cron_signed_url import logging logger = logging.getLogger(__name__) @@ -52,7 +52,6 @@ STATISTICS_IMAGE_HEIGHT = 4 RECURSIVE_LIMIT = 10000 WB_MAX_RETRY = 3 -SITE_KEY = 'rdm_statistics' class InstitutionListViewStat(RdmPermissionMixin, UserPassesTestMixin, TemplateView): """institlutions list view for statistics""" @@ -114,8 +113,6 @@ def get_context_data(self, **kwargs): ctx['current_date'] = current_date ctx['user'] = user ctx['provider_data_array'] = provider_data_array - digest = hashlib.sha512(SITE_KEY.encode('utf-8')).hexdigest() - ctx['token'] = digest.upper() return ctx @@ -448,9 +445,10 @@ class GatherView(TemplateView): raise_exception = True def get(self, request, *args, **kwargs): - # simple authentication - access_token = self.kwargs.get('access_token') - if not simple_auth(access_token): + # signed-URL authentication + ts = self.kwargs.get('ts') + signature = self.kwargs.get('signature') + if not cron_signed_url.verify_signed_params(ts, signature): response_hash = {'state': 'fail', 'error': 'access forbidden'} response_json = json.dumps(response_hash) response = HttpResponse(response_json, content_type='application/json') @@ -590,12 +588,6 @@ def count_project_files(self, node_id, provider, path, cookies): pass self.count_project_files(provider=provider, node_id=node_id, path='/' + path, cookies=cookies) -def simple_auth(access_token): - digest = hashlib.sha512(SITE_KEY.encode('utf-8')).hexdigest() - if digest == access_token.lower(): - return True - else: - return False def send_stat_mail_core(request, **kwargs): """Send statistics information email. diff --git a/admin_tests/rdm_statistics/test_views.py b/admin_tests/rdm_statistics/test_views.py index 97291392f09..57a385c2407 100644 --- a/admin_tests/rdm_statistics/test_views.py +++ b/admin_tests/rdm_statistics/test_views.py @@ -16,6 +16,7 @@ from osf.models.user import Institution from admin.rdm_statistics import views +from framework.auth import cron_signed_url from mock import patch import datetime @@ -181,7 +182,6 @@ def test_get_context_data(self, **kwargs): nt.assert_true('current_date' in ctx) nt.assert_true('user' in ctx) nt.assert_true('provider_data_array' in ctx) - nt.assert_true('token' in ctx) class TestImageView(AdminTestCase): """test ImageView""" @@ -352,9 +352,6 @@ def tearDown(self): self.user.delete() self.institution1.delete() -def test_simple_auth(): - access_key_hexa = '2a85563b2b0f7d3168199f475365f57da1d56e4bb2ce2b7044eb058ae5e287637e7c636a772682d92c8d6b1830b9a97c5a5dc3de7016c60bde4baa7cc3b38aeb' - nt.assert_true(views.simple_auth(access_key_hexa)) def test_get_start_date(): end_date = datetime.datetime.now() @@ -410,7 +407,8 @@ def setUp(self): self.request = RequestFactory().get('/fake_path') self.view = views.GatherView() self.view = setup_user_view(self.view, self.request, user=self.user) - self.view.kwargs = {'institution_id': self.institution1.id, 'access_token': '2A85563B2B0F7D3168199F475365F57DA1D56E4BB2CE2B7044EB058AE5E287637E7C636A772682D92C8D6B1830B9A97C5A5DC3DE7016C60BDE4BAA7CC3B38AEB'.lower()} + ts, signature = cron_signed_url.generate_signed_params() + self.view.kwargs = {'institution_id': self.institution1.id, 'ts': ts, 'signature': signature} def tearDown(self): super(TestGatherView, self).tearDown() @@ -428,6 +426,12 @@ def test_get(self, *args, **kwargs): # metadata addon is now enabled by default, so we have 3 providers nt.assert_equal(len(resp), 3) + def test_get_forbidden_with_invalid_signature(self): + # GatherView.get() reads ts/signature from self.kwargs, set in setUp() + self.view.kwargs['signature'] = 'deadbeef' + resp = json.loads(self.view.get(self.request).content) + nt.assert_equal(resp['state'], 'fail') + def test_send_stat_mail(self, *args, **kwargs): nt.assert_equal(views.send_stat_mail_core(self.request).status_code, 200) diff --git a/framework/auth/cron_signed_url.py b/framework/auth/cron_signed_url.py new file mode 100644 index 00000000000..868e82cf526 --- /dev/null +++ b/framework/auth/cron_signed_url.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +"""HMAC-signed, time-limited URL auth for cron-triggered admin endpoints. + +Used by No.1 (admin/rdm_custom_storage_location/views.py:external_acc_update) +and No.82 (admin/rdm_statistics/views.py:GatherView), which are called by +cron.sh scripts with no interactive user/session. A signed URL looks like +`...///`, where `ts` is a microsecond-resolution Unix +timestamp and `signature` is HMAC(secret, ts). + +Security comes from two independent checks, both required: + 1. authenticity - signature must match HMAC(CRON_SIGNED_URL_SECRET, ts) + 2. freshness - ts must be within CRON_SIGNED_URL_TTL_SECONDS of "now" + +Timestamp precision (microseconds) does not itself add security - it is a +customer requirement, not a defense mechanism. Both checks above are what +actually make an intercepted URL become useless after CRON_SIGNED_URL_TTL_SECONDS. +""" +import hmac +import time + +from website import settings + + +def _sign(ts): + return hmac.new( + key=settings.CRON_SIGNED_URL_SECRET.encode('utf-8'), + msg=str(ts).encode('utf-8'), + digestmod=settings.CRON_SIGNED_URL_HMAC_ALGORITHM, + ).hexdigest() + + +def generate_signed_params(): + """Returns (ts, signature) as strings, ready to drop into a URL path. + + ts is a microsecond-resolution Unix timestamp (str), per customer + requirement (see module docstring). + """ + ts = str(int(time.time() * 1000000)) + return ts, _sign(ts) + + +def verify_signed_params(ts, signature, ttl_seconds=None): + """Verifies a (ts, signature) pair produced by generate_signed_params(). + + Returns True only if signature matches ts AND ts is within ttl_seconds + of now (in either direction). Returns False for any malformed input + instead of raising. + """ + if ttl_seconds is None: + ttl_seconds = settings.CRON_SIGNED_URL_TTL_SECONDS + if not ts or not signature: + return False + try: + ts_us = int(ts) + except (TypeError, ValueError): + return False + expected_signature = _sign(ts) + if not hmac.compare_digest(expected_signature, signature.lower()): + return False + now_us = int(time.time() * 1000000) + ttl_us = int(ttl_seconds * 1000000) + return abs(now_us - ts_us) <= ttl_us diff --git a/osf/management/commands/generate_signed_cron_token.py b/osf/management/commands/generate_signed_cron_token.py new file mode 100644 index 00000000000..98c09cab48f --- /dev/null +++ b/osf/management/commands/generate_signed_cron_token.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from django.core.management.base import BaseCommand + +from framework.auth.cron_signed_url import generate_signed_params + + +class Command(BaseCommand): + """Prints "/" for building a B(2) signed cron URL. + + Used by admin/rdm_statistics/cron.sh (No.82) and + admin/rdm_custom_storage_location/cron.sh (No.1) to mint a fresh + signed URL segment immediately before each curl call. See + framework.auth.cron_signed_url for the verification side. + """ + help = 'Print "/" for a B(2) signed cron URL.' + + def handle(self, *args, **options): + ts, signature = generate_signed_params() + self.stdout.write('{}/{}'.format(ts, signature)) diff --git a/osf_tests/management_commands/test_generate_signed_cron_token.py b/osf_tests/management_commands/test_generate_signed_cron_token.py new file mode 100644 index 00000000000..821d616c75c --- /dev/null +++ b/osf_tests/management_commands/test_generate_signed_cron_token.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +import io +import pytest +from django.core.management import call_command + +from framework.auth import cron_signed_url + + +@pytest.mark.django_db +class TestGenerateSignedCronToken: + + def test_prints_ts_slash_signature(self): + out = io.StringIO() + call_command('generate_signed_cron_token', stdout=out) + output = out.getvalue().strip() + assert output.count('/') == 1 + ts, signature = output.split('/') + assert ts.isdigit() + assert len(signature) > 0 + + def test_printed_pair_verifies_successfully(self): + out = io.StringIO() + call_command('generate_signed_cron_token', stdout=out) + ts, signature = out.getvalue().strip().split('/') + assert cron_signed_url.verify_signed_params(ts, signature) diff --git a/tests/test_cron_signed_url.py b/tests/test_cron_signed_url.py new file mode 100644 index 00000000000..d77169a13ce --- /dev/null +++ b/tests/test_cron_signed_url.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +import time +import mock +from nose.tools import * # noqa (PEP8 asserts) + +from framework.auth import cron_signed_url + + +class TestCronSignedUrl: + + def setup_method(self, method): + self.secret_patch = mock.patch('website.settings.CRON_SIGNED_URL_SECRET', 'test-secret') + self.ttl_patch = mock.patch('website.settings.CRON_SIGNED_URL_TTL_SECONDS', 100) + self.secret_patch.start() + self.ttl_patch.start() + + def teardown_method(self, method): + self.secret_patch.stop() + self.ttl_patch.stop() + + def test_generate_then_verify_succeeds(self): + ts, signature = cron_signed_url.generate_signed_params() + assert_true(cron_signed_url.verify_signed_params(ts, signature)) + + def test_ts_is_microsecond_precision(self): + before_us = int(time.time() * 1000000) + ts, _ = cron_signed_url.generate_signed_params() + after_us = int(time.time() * 1000000) + assert_true(before_us <= int(ts) <= after_us) + + def test_tampered_signature_is_rejected(self): + ts, signature = cron_signed_url.generate_signed_params() + flipped_char = '0' if signature[0] != '0' else '1' + tampered = flipped_char + signature[1:] + assert_false(cron_signed_url.verify_signed_params(ts, tampered)) + + def test_tampered_timestamp_is_rejected(self): + ts, signature = cron_signed_url.generate_signed_params() + tampered_ts = str(int(ts) + 1) + assert_false(cron_signed_url.verify_signed_params(tampered_ts, signature)) + + def test_expired_timestamp_is_rejected(self): + with mock.patch('time.time', return_value=time.time() - 200): + ts, signature = cron_signed_url.generate_signed_params() + assert_false(cron_signed_url.verify_signed_params(ts, signature)) + + def test_future_timestamp_beyond_ttl_is_rejected(self): + with mock.patch('time.time', return_value=time.time() + 200): + ts, signature = cron_signed_url.generate_signed_params() + assert_false(cron_signed_url.verify_signed_params(ts, signature)) + + def test_non_numeric_timestamp_is_rejected(self): + assert_false(cron_signed_url.verify_signed_params('not-a-number', 'deadbeef')) + + def test_missing_signature_is_rejected(self): + ts, _ = cron_signed_url.generate_signed_params() + assert_false(cron_signed_url.verify_signed_params(ts, '')) + + def test_wrong_secret_is_rejected(self): + ts, signature = cron_signed_url.generate_signed_params() + with mock.patch('website.settings.CRON_SIGNED_URL_SECRET', 'a-different-secret'): + assert_false(cron_signed_url.verify_signed_params(ts, signature)) diff --git a/tests/test_oauth.py b/tests/test_oauth.py index 74ce6195df1..af4002d5bf7 100644 --- a/tests/test_oauth.py +++ b/tests/test_oauth.py @@ -38,6 +38,7 @@ ) from django.test import RequestFactory from admin.rdm_custom_storage_location import views as customstoragelocation_update +from framework.auth import cron_signed_url SILENT_LOGGERS = ['oauthlib', 'requests_oauthlib'] @@ -1050,11 +1051,12 @@ def test_callback_with_institution(self, mock_drive_refresh): assert_equal(account.oauth_key, 'mock_access_token') assert_equal(account.provider_id, 'mock_provider_id') req = RequestFactory().get('http://localhost:8001/customstoragelocation/external_acc_update/') - res = customstoragelocation_update.external_acc_update(req,access_token='d610ef95f0b0f5868f13919b8ed64070b9acb9c19b8da9f2c514ed938203ec3e236c9cad4f4146bdf22b4e79cf0d92f6d4f4c996d236c6b0ee79a1336b26afb7') - assert_equal(res.status_code,200) + ts, signature = cron_signed_url.generate_signed_params() + res = customstoragelocation_update.external_acc_update(req, ts=ts, signature=signature) + assert_equal(res.status_code, 200) assert_equal(res.content.decode(), 'Done') - res = customstoragelocation_update.external_acc_update(req,access_token='b610ef95f0b0f5868f13919b8ed64070b9acb9c19b8da9f2c514ed938203ec3e236c9cad4f4146bdf22b4e79cf0d92f6d4f4c996d236c6b0ee79a1336b26afb7') - assert_equal(res.status_code,200) + res = customstoragelocation_update.external_acc_update(req, ts=ts, signature='deadbeef') + assert_equal(res.status_code, 200) assert_not_equal(res.content.decode(), 'Done') @responses.activate diff --git a/website/settings/__init__.py b/website/settings/__init__.py index 38da961d50b..e533b3a4619 100644 --- a/website/settings/__init__.py +++ b/website/settings/__init__.py @@ -21,7 +21,7 @@ if not DEV_MODE: from . import local from . import defaults - for setting in ('WATERBUTLER_JWE_SECRET', 'WATERBUTLER_JWE_SALT', 'WATERBUTLER_JWT_SECRET', 'JWT_SECRET', 'DEFAULT_HMAC_SECRET', 'POPULAR_LINKS_NODE', 'NEW_AND_NOTEWORTHY_LINKS_NODE', 'SENSITIVE_DATA_SALT', 'SENSITIVE_DATA_SECRET'): + for setting in ('WATERBUTLER_JWE_SECRET', 'WATERBUTLER_JWE_SALT', 'WATERBUTLER_JWT_SECRET', 'JWT_SECRET', 'DEFAULT_HMAC_SECRET', 'CRON_SIGNED_URL_SECRET', 'POPULAR_LINKS_NODE', 'NEW_AND_NOTEWORTHY_LINKS_NODE', 'SENSITIVE_DATA_SALT', 'SENSITIVE_DATA_SECRET'): assert getattr(local, setting, None) and getattr(local, setting, None) != getattr(defaults, setting, None), '{} must be specified in local.py when DEV_MODE is False'.format(setting) diff --git a/website/settings/defaults.py b/website/settings/defaults.py index 644f84ba689..1b57ba7516f 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -328,6 +328,13 @@ def parent_dir(path): DEFAULT_HMAC_SECRET = 'changeme' DEFAULT_HMAC_ALGORITHM = hashlib.sha256 + +# signed-URL auth for cron-triggered admin endpoints (No.1 external_acc_update, +# No.82 GatherView). Deliberately separate from DEFAULT_HMAC_SECRET (WaterButler<->OSF) +# so a leak of one does not affect the other. +CRON_SIGNED_URL_SECRET = 'changeme' +CRON_SIGNED_URL_HMAC_ALGORITHM = hashlib.sha256 +CRON_SIGNED_URL_TTL_SECONDS = 100 WATERBUTLER_URL = 'http://localhost:7777' WATERBUTLER_INTERNAL_URL = WATERBUTLER_URL From 370b5f4541c6914dc1b884f103beb1b1228ba047 Mon Sep 17 00:00:00 2001 From: NishikawaKoharu Date: Wed, 16 Sep 2026 13:02:44 +0900 Subject: [PATCH 14/41] =?UTF-8?q?y-websocket=20JWT=E8=AA=8D=E8=A8=BC?= =?UTF-8?q?=E3=81=AECI=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/wiki/tests/test_wiki.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/addons/wiki/tests/test_wiki.py b/addons/wiki/tests/test_wiki.py index 62f48f36531..1d81732dac3 100644 --- a/addons/wiki/tests/test_wiki.py +++ b/addons/wiki/tests/test_wiki.py @@ -3400,8 +3400,10 @@ def test_token_generated_for_editor(self): res = self.app.get(url, auth=self.user.auth) assert_equal(res.status_code, 200) + self.project.reload() body = res.body.decode() sharejs_uuid = get_sharejs_uuid(self.project, self.wname) + assert_true(sharejs_uuid) assert_in(sharejs_uuid, body) token_match = re.search(r'"yWebsocketToken":\s*"([^"]+)"', body) @@ -3420,12 +3422,20 @@ def test_token_not_visible_without_write_permission(self): WikiPage.objects.create_for_node(self.project, self.wname, 'some content', Auth(self.user)) url = self.project.web_url_for('project_wiki_view', wname=self.wname) - res = self.app.get(url) + # Generate sharejs uuid / token path as an editor first (same pattern as TestWikiUuid). + res = self.app.get(url, auth=self.user.auth) assert_equal(res.status_code, 200) + self.project.reload() + sharejs_uuid = get_sharejs_uuid(self.project, self.wname) + assert_true(sharejs_uuid) + # Users without write permission should not receive uuid or a non-empty token. + res = self.app.get(url) + assert_equal(res.status_code, 200) body = res.body.decode() - assert_not_in(get_sharejs_uuid(self.project, self.wname), body) - assert_not_in('"yWebsocketToken":', body) + assert_not_in(sharejs_uuid, body) + token_match = re.search(r'"yWebsocketToken":\s*"([^"]+)"', body) + assert_false(token_match) def test_generate_y_websocket_token_without_secret(self): with mock.patch('addons.wiki.settings.Y_WEBSOCKET_SECRET', ''): From 15c2cf91fa7f3e5bddafcaf7a6096e2e0d44f26f Mon Sep 17 00:00:00 2001 From: NishikawaKoharu Date: Wed, 16 Sep 2026 14:46:04 +0900 Subject: [PATCH 15/41] =?UTF-8?q?y-websocket=20JWT=E8=AA=8D=E8=A8=BC?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=81=AEmock=E3=82=92setUp=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/wiki/tests/test_wiki.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/addons/wiki/tests/test_wiki.py b/addons/wiki/tests/test_wiki.py index 1d81732dac3..3646235013f 100644 --- a/addons/wiki/tests/test_wiki.py +++ b/addons/wiki/tests/test_wiki.py @@ -3382,11 +3382,20 @@ def test_project_update_wiki_page_sort(self): self.assertEqual(result_wiki_child_page3, {'parent_id': wiki_child_page2_id, 'sort_order': 1}) @pytest.mark.enable_bookmark_creation -@mock.patch('addons.wiki.settings.Y_WEBSOCKET_SECRET', 'test-y-websocket-secret') class TestYWebsocketToken(OsfTestCase): def setUp(self): super(TestYWebsocketToken, self).setUp() + # Class-level @mock.patch is unreliable under pytest here; patch in setUp + # so generate_y_websocket_token sees a non-empty secret during the request. + self._y_websocket_secret = 'test-y-websocket-secret' + self._secret_patcher = mock.patch( + 'addons.wiki.settings.Y_WEBSOCKET_SECRET', + self._y_websocket_secret, + ) + self._secret_patcher.start() + self.addCleanup(self._secret_patcher.stop) + self.user = AuthUserFactory() self.project = ProjectFactory(is_public=True, creator=self.user) self.wname = 'foo.bar' @@ -3413,7 +3422,7 @@ def test_token_generated_for_editor(self): payload = jwt.decode( token, - wiki_settings.Y_WEBSOCKET_SECRET, + self._y_websocket_secret, algorithms=[wiki_settings.Y_WEBSOCKET_JWT_ALGORITHM], ) assert_equal(payload['doc_id'], sharejs_uuid) From 0c1040848e4a60a04e05924e1c0151d69f5b33bd Mon Sep 17 00:00:00 2001 From: NishikawaKoharu Date: Wed, 16 Sep 2026 16:10:14 +0900 Subject: [PATCH 16/41] =?UTF-8?q?y-websocket=20JWT=E8=AA=8D=E8=A8=BC?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=81=AE=E3=83=88=E3=83=BC=E3=82=AF?= =?UTF-8?q?=E3=83=B3=E6=A4=9C=E5=87=BAregex=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/wiki/tests/test_wiki.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/addons/wiki/tests/test_wiki.py b/addons/wiki/tests/test_wiki.py index 3646235013f..9c298339445 100644 --- a/addons/wiki/tests/test_wiki.py +++ b/addons/wiki/tests/test_wiki.py @@ -3415,7 +3415,8 @@ def test_token_generated_for_editor(self): assert_true(sharejs_uuid) assert_in(sharejs_uuid, body) - token_match = re.search(r'"yWebsocketToken":\s*"([^"]+)"', body) + # edit.mako renders `yWebsocketToken: "..."`, not JSON `"yWebsocketToken": "..."`. + token_match = re.search(r'yWebsocketToken:\s*"([^"]+)"', body) assert_true(token_match) token = token_match.group(1) assert_true(token) @@ -3443,7 +3444,7 @@ def test_token_not_visible_without_write_permission(self): assert_equal(res.status_code, 200) body = res.body.decode() assert_not_in(sharejs_uuid, body) - token_match = re.search(r'"yWebsocketToken":\s*"([^"]+)"', body) + token_match = re.search(r'yWebsocketToken:\s*"([^"]+)"', body) assert_false(token_match) def test_generate_y_websocket_token_without_secret(self): From 47424ede3a05d3e4e71db8049969e5f846287e73 Mon Sep 17 00:00:00 2001 From: ndnhat Date: Tue, 22 Sep 2026 11:12:28 +0700 Subject: [PATCH 17/41] =?UTF-8?q?Ref:=20=E6=A9=9F=E9=96=A2=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E8=80=85=E6=A9=9F=E8=83=BDAPI=E3=81=AE=E9=81=A9?= =?UTF-8?q?=E5=88=87=E3=81=AA=E6=A8=A9=E9=99=90=E4=BB=98=E4=B8=8E:=20Fix?= =?UTF-8?q?=20error=20when=20access=20statistic/index?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- admin/rdm_statistics/views.py | 2 +- admin_tests/rdm_statistics/test_views.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/admin/rdm_statistics/views.py b/admin/rdm_statistics/views.py index 9c8f89c2995..20f8dc23017 100644 --- a/admin/rdm_statistics/views.py +++ b/admin/rdm_statistics/views.py @@ -791,7 +791,7 @@ def test_func(self): def find_bookmark_collection(self, user): collection = apps.get_model('osf.Collection') - return collection.objects.get(creator=user, is_deleted=False, is_bookmark_collection=True) + return collection.objects.get(creator=user, deleted__isnull=True, is_bookmark_collection=True) def get(self, request, *args, **kwargs): user = self.request.user diff --git a/admin_tests/rdm_statistics/test_views.py b/admin_tests/rdm_statistics/test_views.py index 57a385c2407..181c64d51ce 100644 --- a/admin_tests/rdm_statistics/test_views.py +++ b/admin_tests/rdm_statistics/test_views.py @@ -7,6 +7,7 @@ from tests.base import AdminTestCase from osf_tests.factories import ( AuthUserFactory, + BookmarkCollectionFactory, InstitutionFactory, ProjectFactory ) @@ -568,9 +569,6 @@ def test_denied_for_anonymous(self): views.IndexView.as_view()(request) def test_allowed_for_super_admin(self): - # Only test_func (permission check) is exercised here, not the full - # get() flow — IndexView.find_bookmark_collection is currently broken - # and fixing it is out of scope for this change. request = RequestFactory().get('/fake_path') request.user = self.superuser view = views.IndexView() @@ -578,11 +576,16 @@ def test_allowed_for_super_admin(self): nt.assert_true(view.test_func()) def test_allowed_for_institution_admin(self): - # Only test_func (permission check) is exercised here, not the full - # get() flow — IndexView.find_bookmark_collection is currently broken - # and fixing it is out of scope for this change. request = RequestFactory().get('/fake_path') request.user = self.institution_admin view = views.IndexView() view.request = request nt.assert_true(view.test_func()) + + def test_find_bookmark_collection(self): + """Collection has no 'is_deleted' field; filtering on it raised FieldError.""" + # conftest mocks out new_bookmark_collection for speed, so the bookmark + # collection normally created with the user has to be added here. + bookmark_collection = BookmarkCollectionFactory(creator=self.superuser) + view = views.IndexView() + nt.assert_equal(view.find_bookmark_collection(self.superuser), bookmark_collection) From 1c3801e35c21d43397ead6bb1b40c56e88533924 Mon Sep 17 00:00:00 2001 From: hcphat Date: Wed, 23 Sep 2026 09:12:27 +0700 Subject: [PATCH 18/41] =?UTF-8?q?ref=202.2.=E3=82=A2=E3=83=89=E3=82=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AB?= =?UTF-8?q?=E7=B5=A1=E3=82=80=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E6=99=82=E3=81=AE=E3=82=AF=E3=82=AA=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=AE=A1=E7=90=86:=20Update=20file=20size=20logic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_views_copy_move_quota.py | 84 +++++++++++++++++++ addons/osfstorage/views.py | 46 ++++++++-- tests/test_quota_move_copy.py | 30 ++++++- website/util/quota.py | 9 +- 4 files changed, 159 insertions(+), 10 deletions(-) diff --git a/addons/osfstorage/tests/test_views_copy_move_quota.py b/addons/osfstorage/tests/test_views_copy_move_quota.py index 2364b770363..4948d7c8770 100644 --- a/addons/osfstorage/tests/test_views_copy_move_quota.py +++ b/addons/osfstorage/tests/test_views_copy_move_quota.py @@ -323,6 +323,58 @@ def test_copy_folder_fallback_when_fileinfo_missing(self, mock_st, mock_uq): for fid in cloned_ids: assert FileInfo.objects.filter(file_id=fid).exists() + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.osfstorage.views.logger') + def test_copy_folder_fallback_none_size_logs_warning_and_sets_zero(self, mock_logger, mock_st, mock_uq): + """ + When the source file has no FileInfo + AND its latest version has size=None, the clone's FileInfo must be 0 + and a warning must be logged. + """ + src_folder = self.root_node.append_folder('none_size_folder') + v = factories.FileVersionFactory(size=None) + f = src_folder.append_file('none_size.txt') + f.add_version(v); f.save() + # Intentionally do NOT create a FileInfo for f + + dest_folder = self.root_node.append_folder('dest_none_size') + resp = self._copy_folder(src_folder, dest_folder) + assert resp.status_code == 201 + + cloned = dest_folder.find_child_by_name('none_size_folder') + from addons.osfstorage.views import _get_all_descendant_file_ids + cloned_ids = _get_all_descendant_file_ids(cloned) + for fid in cloned_ids: + fi = FileInfo.objects.filter(file_id=fid).first() + assert fi is not None + assert fi.file_size == 0 + assert mock_logger.warning.called + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.osfstorage.views.logger') + def test_copy_folder_fallback_negative_size_logs_warning_and_sets_zero(self, mock_logger, mock_st, mock_uq): + """ A negative version size (-1 ="not yet set") must be treated as 0 with a warning """ + src_folder = self.root_node.append_folder('neg_size_folder') + v = factories.FileVersionFactory(size=-1) + f = src_folder.append_file('neg_size.txt') + f.add_version(v); f.save() + # Intentionally do NOT create a FileInfo for f + + dest_folder = self.root_node.append_folder('dest_neg_size') + resp = self._copy_folder(src_folder, dest_folder) + assert resp.status_code == 201 + + cloned = dest_folder.find_child_by_name('neg_size_folder') + from addons.osfstorage.views import _get_all_descendant_file_ids + cloned_ids = _get_all_descendant_file_ids(cloned) + for fid in cloned_ids: + fi = FileInfo.objects.filter(file_id=fid).first() + assert fi is not None + assert fi.file_size == 0 + assert mock_logger.warning.called + @mock.patch('addons.osfstorage.views.update_quota') @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) @mock.patch('addons.osfstorage.views._get_all_descendant_file_ids') @@ -833,6 +885,38 @@ def test_upload_same_size_no_quota_update(self, mock_st, mock_uq): # delta == 0 → update_quota should NOT be called assert not mock_uq.called + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.osfstorage.views.logger') + def test_upload_none_size_logs_warning_and_sets_zero(self, mock_logger, mock_st, mock_uq): + """ + When WaterButler reports size=None for a new upload, FileInfo.file_size + must be 0 and a warning must be logged. + """ + res = self._upload(self.root_node, 'none_size.txt', size=None) + assert res.status_code in (200, 201) + record = self.root_node.find_child_by_name('none_size.txt') + fi = FileInfo.objects.filter(file=record).first() + assert fi is not None + assert fi.file_size == 0 + assert mock_logger.warning.called + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.osfstorage.views.logger') + def test_upload_negative_size_logs_warning_and_sets_zero(self, mock_logger, mock_st, mock_uq): + """ + A negative size (-1 = "not yet set") must be treated as 0 with a + warning, not stored as-is in FileInfo.file_size. + """ + res = self._upload(self.root_node, 'neg_size.txt', size=-1) + assert res.status_code in (200, 201) + record = self.root_node.find_child_by_name('neg_size.txt') + fi = FileInfo.objects.filter(file=record).first() + assert fi is not None + assert fi.file_size == 0 + assert mock_logger.warning.called + # --------------------------------------------------------------------------- # osfstorage_delete – folder quota subtraction diff --git a/addons/osfstorage/views.py b/addons/osfstorage/views.py index 30203cf57e0..3c3715ab636 100644 --- a/addons/osfstorage/views.py +++ b/addons/osfstorage/views.py @@ -159,7 +159,15 @@ def osfstorage_copy_hook(source, destination, name=None, **kwargs): if getattr(source.target, 'type', '') == 'osf.node' and getattr(destination.target, 'type', '') == 'osf.node': if source.is_file: latest_version = cloned.versions.order_by('-created').first() - new_size = latest_version.size if latest_version and latest_version.size else 0 + if latest_version and latest_version.size is not None and latest_version.size > 0: + new_size = latest_version.size + else: + if latest_version and (latest_version.size is None or latest_version.size < 0): + logger.warning( + 'latest_version.size is %r for cloned file %s (osfstorage_copy_hook); ' + 'treating size as 0 for quota calculation', latest_version.size, cloned._id + ) + new_size = 0 FileInfo.objects.update_or_create(file=cloned, defaults={'file_size': new_size}) else: source_file_ids = _get_all_descendant_file_ids(source) @@ -185,7 +193,15 @@ def osfstorage_copy_hook(source, destination, name=None, **kwargs): size = 0 if node: v = node.versions.order_by('-created').first() - size = (v.size or 0) if v else 0 + if v and v.size is not None and v.size > 0: + size = v.size + else: + if v and (v.size is None or v.size < 0): + logger.warning( + 'v.size is %r for source file %s (osfstorage_copy_hook FOLDER branch); ' + 'treating size as 0 for quota calculation', v.size, src_id + ) + size = 0 else: # src_id is None and not in map size = 0 @@ -232,7 +248,15 @@ def osfstorage_move_hook(source, destination, name=None, **kwargs): if getattr(source_target, 'type', '') == 'osf.node' and getattr(destination.target, 'type', '') == 'osf.node': if source.is_file: latest_version = source.versions.order_by('-created').first() - new_size = latest_version.size if latest_version and latest_version.size else 0 + if latest_version and latest_version.size is not None and latest_version.size > 0: + new_size = latest_version.size + else: + if latest_version and (latest_version.size is None or latest_version.size < 0): + logger.warning( + 'latest_version.size is %r for moved file %s (osfstorage_move_hook); ' + 'treating size as 0 for quota calculation', latest_version.size, source._id + ) + new_size = 0 # FileInfo: UPSERT FileInfo.objects.update_or_create(file=source, defaults={'file_size': new_size}) else: @@ -502,10 +526,18 @@ def osfstorage_create_child(file_node, payload, **kwargs): # Only handle if target is a node if getattr(file_node.target, 'type', '') == 'osf.node': - new_size = new_version.size if new_version.size and new_version.size > 0 else 0 - if new_size >= 0: - old_info = FileInfo.objects.filter(file=file_node).first() - old_size = old_info.file_size if old_info else 0 + if new_version.size is not None and new_version.size > 0: + new_size = new_version.size + else: + if new_version.size is None or new_version.size < 0: + logger.warning( + 'new_version.size is %r for created file %s (osfstorage_create_child); ' + 'treating size as 0 for quota calculation', new_version.size, file_node._id + ) + new_size = 0 + + old_info = FileInfo.objects.filter(file=file_node).first() + old_size = old_info.file_size if old_info else 0 # FileInfo: UPSERT FileInfo.objects.update_or_create( diff --git a/tests/test_quota_move_copy.py b/tests/test_quota_move_copy.py index 114113ebbf3..10ccf502c09 100644 --- a/tests/test_quota_move_copy.py +++ b/tests/test_quota_move_copy.py @@ -32,12 +32,38 @@ def test_get_file_size_nested(): } assert quota.get_file_size(children) == 150 +@mock.patch('website.util.quota.logger') +def test_get_file_size_none_size_treated_as_zero_and_warns(mock_logger): + """A child with size=None must not crash get_file_size.""" + children = {'children': [{'kind': 'file', 'size': None, 'name': 'unknown.gdoc'}]} + assert quota.get_file_size(children) == 0 + assert mock_logger.warning.called + +@mock.patch('website.util.quota.logger') +def test_get_file_size_negative_size_treated_as_zero_and_warns(mock_logger): + """A negative child size must be treated as 0.""" + children = {'children': [{'kind': 'file', 'size': -1, 'name': 'bad.txt'}]} + assert quota.get_file_size(children) == 0 + assert mock_logger.warning.called + +@mock.patch('website.util.quota.logger') +def test_get_file_size_mixed_children_none_and_valid(mock_logger): + """A None-size sibling must not swallow the total contributed by valid siblings.""" + children = { + 'children': [ + {'kind': 'file', 'size': 100, 'name': 'ok.txt'}, + {'kind': 'file', 'size': None, 'name': 'unknown.gdoc'}, + ] + } + assert quota.get_file_size(children) == 100 + assert mock_logger.warning.called + @mock.patch('website.util.quota.ProjectStorageType.objects.get') @mock.patch('website.util.quota.AbstractNode.objects.get') @mock.patch('website.util.quota.update_quota') @pytest.mark.django_db def test_handle_move_copy_source_extend_dest_osfstorage(mock_update_quota, mock_node_get, mock_pst_get, mock_node): - """source=s3compatinstitutions, dest=osfstorage → update_quota không được gọi vì source không phải osfstorage""" + """source=s3compatinstitutions, dest=osfstorage""" mock_pst_get.return_value = mock.Mock(storage_type=1) mock_node_get.return_value = mock_node @@ -59,7 +85,7 @@ def test_handle_move_copy_source_osfstorage(mock_update_quota, mock_node_get, mo mock_node_get.return_value = mock_node payload = { - 'destination': {'provider': 's3compatinstitutions', 'size': 123}, # <-- thêm size + 'destination': {'provider': 's3compatinstitutions', 'size': 123}, 'source': {'provider': 'osfstorage', 'size': 123, 'nid': 'nid1'} } quota._handle_move_copy(quota.FileLog.FILE_MOVED, mock_node, mock_user, payload) diff --git a/website/util/quota.py b/website/util/quota.py index 31688511674..1b5098304a8 100644 --- a/website/util/quota.py +++ b/website/util/quota.py @@ -468,7 +468,14 @@ def get_file_size(children): size = 0 for child in children: if child['kind'] == 'file': - size += int(child['size']) + child_size = child['size'] + if child_size is None or child_size < 0: + logger.warning( + 'child size is %r for %r; treating size as 0 for quota calculation', + child_size, child.get('name') + ) + child_size = 0 + size += int(child_size) else: size += get_file_size(child) return size From 382a0ac8ae95366f5649d36f26061decd6100cd0 Mon Sep 17 00:00:00 2001 From: NishikawaKoharu Date: Thu, 24 Sep 2026 18:10:05 +0900 Subject: [PATCH 19/41] =?UTF-8?q?Wiki=E3=82=A4=E3=83=B3=E3=83=9D=E3=83=BC?= =?UTF-8?q?=E3=83=88=E3=82=BF=E3=82=B9=E3=82=AF=E7=B5=90=E6=9E=9CAPI?= =?UTF-8?q?=E3=81=A7task=5Fid=E3=81=A8=E3=83=8E=E3=83=BC=E3=83=89=E3=81=AE?= =?UTF-8?q?=E5=AF=BE=E5=BF=9C=E3=82=92=E6=A4=9C=E8=A8=BC=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/wiki/tests/test_wiki.py | 50 +++++++++++++++++++++++++++ addons/wiki/tests/test_wiki_import.py | 7 ++++ addons/wiki/views.py | 38 ++++++++++++++++++-- 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/addons/wiki/tests/test_wiki.py b/addons/wiki/tests/test_wiki.py index a7d37cb5529..f946f499361 100644 --- a/addons/wiki/tests/test_wiki.py +++ b/addons/wiki/tests/test_wiki.py @@ -1866,6 +1866,16 @@ def test_project_wiki_validate_for_import(self, mock_check_file_object_in_node): task_id = response_json['taskId'] uuid_obj = uuid.UUID(task_id) assert uuid_obj + task = WikiImportTask.objects.get(task_id=task_id) + self.assertEqual(task.node, self.project) + self.assertEqual(task.status, WikiImportTask.STATUS_COMPLETED) + self.assertIsNotNone(task.process_end) + self.assertEqual( + WikiImportTask.objects.filter( + node=self.project, status=WikiImportTask.STATUS_RUNNING + ).count(), + 0, + ) @mock.patch('addons.wiki.utils.check_file_object_in_node') def test_project_wiki_validate_for_import_error(self, mock_check_file_object_in_node): @@ -1964,6 +1974,9 @@ def test_project_wiki_import(self, mock_check_file_object_in_node, mock_project_ response_json = res.json task_id = response_json['taskId'] uuid_obj = uuid.UUID(task_id) + task = WikiImportTask.objects.get(task_id=task_id) + self.assertEqual(task.node, self.project) + self.assertEqual(task.status, WikiImportTask.STATUS_RUNNING) @mock.patch('addons.wiki.utils.check_file_object_in_node') def test_project_wiki_import_error(self, mock_check_file_object_in_node): @@ -1983,6 +1996,43 @@ def test_project_wiki_import_with_no_admin_permission(self): res = self.app.post_json(url, { 'data': [{'test': 'test1'}] }, expect_errors=True) assert_equal(res.status_code, 401) + @mock.patch('addons.wiki.views.AsyncResult') + def test_project_get_task_result_with_admin_permission(self, mock_async_result): + mock_res = mock.MagicMock() + mock_res.ready.return_value = True + mock_res.get.return_value = {'canStartImport': True, 'data': []} + mock_async_result.return_value = mock_res + WikiImportTask.objects.create( + node=self.project, + task_id='task-id', + status=WikiImportTask.STATUS_COMPLETED, + creator=self.user, + ) + url = self.project.api_url_for('project_get_task_result', task_id='task-id') + res = self.app.get(url, auth=self.user.auth) + assert_equal(res.status_code, http_status.HTTP_200_OK) + assert_equal(res.json, {'canStartImport': True, 'data': []}) + + @mock.patch('addons.wiki.views.AsyncResult') + def test_project_get_task_result_rejects_task_for_other_node(self, mock_async_result): + other_project = ProjectFactory(creator=self.user) + WikiImportTask.objects.create( + node=other_project, + task_id='other-node-task-id', + status=WikiImportTask.STATUS_COMPLETED, + creator=self.user, + ) + url = self.project.api_url_for('project_get_task_result', task_id='other-node-task-id') + res = self.app.get(url, auth=self.user.auth, expect_errors=True) + assert_equal(res.status_code, http_status.HTTP_404_NOT_FOUND) + mock_async_result.assert_not_called() + + def test_project_get_task_result_rejects_unknown_task_id(self): + url = self.project.api_url_for('project_get_task_result', task_id='unknown-task-id') + res = self.app.get(url, auth=self.user.auth, expect_errors=True) + assert_equal(res.status_code, http_status.HTTP_404_NOT_FOUND) + + @mock.patch('celery.contrib.abortable.AbortableAsyncResult') def test_wiki_import_create_or_update_aborted(self, mock_task): mock_task.is_aborted.return_value = True diff --git a/addons/wiki/tests/test_wiki_import.py b/addons/wiki/tests/test_wiki_import.py index f17fc8dc6d3..b57bf6545c4 100644 --- a/addons/wiki/tests/test_wiki_import.py +++ b/addons/wiki/tests/test_wiki_import.py @@ -1250,6 +1250,10 @@ def test_project_wiki_validate_for_import(self, mock_check_file_object_in_node): task_id = response_json['taskId'] uuid_obj = uuid.UUID(task_id) assert uuid_obj + task = WikiImportTask.objects.get(task_id=task_id) + assert_equal(task.node, self.project) + assert_equal(task.status, WikiImportTask.STATUS_COMPLETED) + assert_is_not_none(task.process_end) def test_project_wiki_validate_for_import_process(self): result = views.project_wiki_validate_for_import_process( @@ -1323,6 +1327,9 @@ def test_project_wiki_import(self, mock_check_file_object_in_node, mock_project_ task_id = response_json['taskId'] uuid_obj = uuid.UUID(task_id) assert_is_not_none(uuid_obj) + task = WikiImportTask.objects.get(task_id=task_id) + assert_equal(task.node, self.project) + assert_equal(task.status, WikiImportTask.STATUS_RUNNING) @mock.patch('addons.wiki.views._get_md_content_from_wb') @mock.patch('addons.wiki.views._get_or_create_wiki_folder') diff --git a/addons/wiki/views.py b/addons/wiki/views.py index afebea3cd01..ca5fda4ebbb 100644 --- a/addons/wiki/views.py +++ b/addons/wiki/views.py @@ -771,15 +771,40 @@ def serialize_component_wiki(node, auth): return component return None + +def _bind_wiki_task_to_node(task_id, node, auth, status, set_process_end=False): + """Record task_id as belonging to node for get_task_result authorization. + + Validate tasks must not use STATUS_RUNNING so they do not affect + "importing" UI, abort, or concurrent-import checks. + """ + defaults = { + 'node': node, + 'status': status, + 'creator': auth.user if auth else None, + } + if set_process_end: + defaults['process_end'] = timezone.make_naive(timezone.now(), timezone.utc) + WikiImportTask.objects.get_or_create(task_id=task_id, defaults=defaults) + + +def _wiki_task_belongs_to_node(task_id, node): + return WikiImportTask.objects.filter(task_id=task_id, node=node).exists() + + @must_be_valid_project @must_have_permission(ADMIN) @must_not_be_registration @must_have_addon('wiki', 'node') -def project_wiki_validate_for_import(dir_id, node, **kwargs): +def project_wiki_validate_for_import(dir_id, auth, node, **kwargs): wiki_utils.check_file_object_in_node(dir_id, node) node_id = node.guids.first()._id task = tasks.run_project_wiki_validate_for_import.delay(dir_id, node_id) task_id = task.id + # Completed + process_end: ownership only; not treated as a running import. + _bind_wiki_task_to_node( + task_id, node, auth, WikiImportTask.STATUS_COMPLETED, set_process_end=True + ) return {'taskId': task_id} def project_wiki_validate_for_import_process(dir_id, node): @@ -907,11 +932,15 @@ def project_wiki_import(dir_id, auth, node, **kwargs): data_json = json.dumps(data) task = tasks.run_project_wiki_import.delay(data_json, dir_id, current_user_id, node_id) task_id = task.id + # Bind before returning so polling get_task_result does not race the worker. + _bind_wiki_task_to_node(task_id, node, auth, WikiImportTask.STATUS_RUNNING) + check_running_task(task_id, node) return {'taskId': task_id} def project_wiki_import_process(data, dir_id, task_id, auth, node): logger.info('----WIKI IMPORT DIRECTORY_ID: {}, PROJECT_NAME: {} ----'.format(dir_id, node.title)) - WikiImportTask.objects.create(node=node, task_id=task_id, status=WikiImportTask.STATUS_RUNNING, creator=auth.user) + # May already exist if project_wiki_import bound the task before the worker ran. + _bind_wiki_task_to_node(task_id, node, auth, WikiImportTask.STATUS_RUNNING) check_running_task(task_id, node) ret = [] wiki_id_list = [] @@ -1313,6 +1342,11 @@ def _create_import_error_list(wiki_infos, imported_list): @must_not_be_registration @must_have_addon('wiki', 'node') def project_get_task_result(task_id, node, **kwargs): + if not _wiki_task_belongs_to_node(task_id, node): + raise HTTPError(http_status.HTTP_404_NOT_FOUND, data=dict( + message_short='Not found', + message_long='Task not found.', + )) res = AsyncResult(task_id, app=celery_app) result = None if not res.ready(): From c194e79d11cbd89ea1691b072c01e738796ba113 Mon Sep 17 00:00:00 2001 From: NishikawaKoharu Date: Fri, 25 Sep 2026 14:45:34 +0900 Subject: [PATCH 20/41] =?UTF-8?q?y-websocket=20JWT=E3=81=AB=E5=88=A9?= =?UTF-8?q?=E7=94=A8=E8=80=85=E8=AD=98=E5=88=A5=E5=AD=90sub=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/wiki/tests/test_wiki.py | 7 ++++++- addons/wiki/utils.py | 10 +++++++--- addons/wiki/views.py | 5 ++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/addons/wiki/tests/test_wiki.py b/addons/wiki/tests/test_wiki.py index 9c298339445..885a6e82a81 100644 --- a/addons/wiki/tests/test_wiki.py +++ b/addons/wiki/tests/test_wiki.py @@ -3427,6 +3427,7 @@ def test_token_generated_for_editor(self): algorithms=[wiki_settings.Y_WEBSOCKET_JWT_ALGORITHM], ) assert_equal(payload['doc_id'], sharejs_uuid) + assert_equal(payload['sub'], self.user._id) def test_token_not_visible_without_write_permission(self): WikiPage.objects.create_for_node(self.project, self.wname, 'some content', Auth(self.user)) @@ -3449,5 +3450,9 @@ def test_token_not_visible_without_write_permission(self): def test_generate_y_websocket_token_without_secret(self): with mock.patch('addons.wiki.settings.Y_WEBSOCKET_SECRET', ''): - assert_equal(generate_y_websocket_token('doc-id'), '') + assert_equal(generate_y_websocket_token('doc-id', 'user-id'), '') + + def test_generate_y_websocket_token_without_user_id(self): + assert_equal(generate_y_websocket_token('doc-id', ''), '') + assert_equal(generate_y_websocket_token('doc-id', None), '') diff --git a/addons/wiki/utils.py b/addons/wiki/utils.py index 51304a4410c..947cf31b834 100644 --- a/addons/wiki/utils.py +++ b/addons/wiki/utils.py @@ -68,17 +68,21 @@ def get_sharejs_uuid(node, wname): )) if private_uuid else None -def generate_y_websocket_token(doc_id): +def generate_y_websocket_token(doc_id, user_id): """ Generate a signed JWT for y-websocket connection authorization. - Returns an empty string when Y_WEBSOCKET_SECRET is not configured. + + Includes ``sub`` (OSF user GUID) for connection traceability on the + y-websocket side. Returns an empty string when Y_WEBSOCKET_SECRET is not + configured or required claims are missing. """ secret = wiki_settings.Y_WEBSOCKET_SECRET - if not secret or not doc_id: + if not secret or not doc_id or not user_id: return '' payload = { 'doc_id': doc_id, + 'sub': user_id, 'exp': timezone.now() + datetime.timedelta(seconds=wiki_settings.Y_WEBSOCKET_TOKEN_TTL), } token = jwt.encode( diff --git a/addons/wiki/views.py b/addons/wiki/views.py index 952daf12838..8d8c25defa8 100644 --- a/addons/wiki/views.py +++ b/addons/wiki/views.py @@ -330,7 +330,10 @@ def project_wiki_view(auth, wname, path=None, **kwargs): if wiki_key not in node.wiki_private_uuids: wiki_utils.generate_private_uuid(node, wiki_name) sharejs_uuid = wiki_utils.get_sharejs_uuid(node, wiki_name) - y_websocket_token = wiki_utils.generate_y_websocket_token(sharejs_uuid) + y_websocket_token = wiki_utils.generate_y_websocket_token( + sharejs_uuid, + auth.user._id, + ) else: if not wiki_page and wiki_key != 'home': raise WIKI_PAGE_NOT_FOUND_ERROR From 6bbcb08b572088b0976590417d007429b4a27d65 Mon Sep 17 00:00:00 2001 From: hcphat Date: Mon, 28 Sep 2026 14:21:14 +0700 Subject: [PATCH 21/41] =?UTF-8?q?ref=202.2.=E3=82=A2=E3=83=89=E3=82=AA?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8=E3=81=AB?= =?UTF-8?q?=E7=B5=A1=E3=82=80=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E6=99=82=E3=81=AE=E3=82=AF=E3=82=AA=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=AE=A1=E7=90=86:=20Update=20handle=20size=20is=20string?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_quota_move_copy.py | 13 +++++++++++++ website/util/quota.py | 5 ++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_quota_move_copy.py b/tests/test_quota_move_copy.py index 10ccf502c09..49081f7ad72 100644 --- a/tests/test_quota_move_copy.py +++ b/tests/test_quota_move_copy.py @@ -46,6 +46,19 @@ def test_get_file_size_negative_size_treated_as_zero_and_warns(mock_logger): assert quota.get_file_size(children) == 0 assert mock_logger.warning.called +def test_get_file_size_string_size(): + """Providers that report size as a str (e.g. s3compat HEAD Content-Length header) + must not raise TypeError when compared against 0.""" + children = {'children': [{'kind': 'file', 'size': '100'}, {'kind': 'file', 'size': '200'}]} + assert quota.get_file_size(children) == 300 + +@mock.patch('website.util.quota.logger') +def test_get_file_size_negative_string_size_treated_as_zero_and_warns(mock_logger): + """A negative child size reported as a str must also be treated as 0.""" + children = {'children': [{'kind': 'file', 'size': '-1', 'name': 'bad.txt'}]} + assert quota.get_file_size(children) == 0 + assert mock_logger.warning.called + @mock.patch('website.util.quota.logger') def test_get_file_size_mixed_children_none_and_valid(mock_logger): """A None-size sibling must not swallow the total contributed by valid siblings.""" diff --git a/website/util/quota.py b/website/util/quota.py index 1b5098304a8..d38d3664e4b 100644 --- a/website/util/quota.py +++ b/website/util/quota.py @@ -468,7 +468,10 @@ def get_file_size(children): size = 0 for child in children: if child['kind'] == 'file': - child_size = child['size'] + try: + child_size = int(child['size']) + except (TypeError, ValueError): + child_size = None if child_size is None or child_size < 0: logger.warning( 'child size is %r for %r; treating size as 0 for quota calculation', From a3325d0c5317f536e7eaf1204110e1b0cda3ce52 Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Fri, 2 Oct 2026 15:26:40 +0900 Subject: [PATCH 22/41] feat(s3): sign S3 addon requests with SigV4 Replace the boto 2 client with boto3/botocore so that the S3 addon can talk to regions that only accept Signature Version 4 (ap-northeast-1 and later). - utils.py / views.py / models.py build the client through boto3 with a region-specific endpoint, and surface botocore exceptions as HTTPError - folder_id becomes "bucket:/prefix/" so that a subfolder can be selected; the legacy "bucket" form is still accepted - the node/user config JS, the Rubeus config and the log action lists follow the new folder shape - requirements.txt gains boto3/botocore; addons/s3/requirements.txt is kept as an empty file because the Jenkins Dockerfile still does COPY ./addons/s3/requirements.txt and would fail the image build without it - MAX_UPLOAD_SIZE stays at 50GB: multipart upload handles up to 5TB, so the lower limit the first draft introduced was unnecessary - the ${ _("...") } gettext wrappers are kept in all three mako templates (20 translatable strings); only the aria-label/alt attributes and the th class change are new Co-Authored-By: An Qiuyu --- Dockerfile | 1 - addons/s3/README.md | 20 ++ addons/s3/apps.py | 1 + addons/s3/models.py | 85 ++++--- addons/s3/provider.py | 4 +- addons/s3/requirements.txt | 3 +- addons/s3/settings/__init__.py | 2 +- addons/s3/static/node-cfg.js | 2 +- addons/s3/static/s3-rubeus-cfg.js | 83 ++++--- .../s3/static/s3AnonymousLogActionList.json | 20 +- addons/s3/static/s3LogActionList.json | 20 +- addons/s3/static/s3NodeConfig.js | 112 +++++----- addons/s3/static/s3UserConfig.js | 46 ++-- addons/s3/static/settings.json | 2 +- addons/s3/static/user-cfg.js | 2 +- addons/s3/templates/s3_credentials_modal.mako | 2 +- addons/s3/templates/s3_node_settings.mako | 6 +- addons/s3/templates/s3_user_settings.mako | 6 +- addons/s3/tests/factories.py | 7 +- addons/s3/tests/test_model.py | 37 +-- addons/s3/tests/test_serializer.py | 7 +- addons/s3/tests/test_view.py | 211 +++++++++--------- addons/s3/tests/utils.py | 1 - addons/s3/utils.py | 130 +++++++---- addons/s3/views.py | 21 +- requirements.txt | 2 + 26 files changed, 460 insertions(+), 373 deletions(-) diff --git a/Dockerfile b/Dockerfile index f23a48e31cb..08e0fc07d22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,6 @@ COPY ./addons/onedrive/requirements.txt /code/addons/onedrive/ COPY ./addons/onedrivebusiness/requirements.txt /code/addons/onedrivebusiness/ #COPY ./addons/osfstorage/requirements.txt ./addons/osfstorage/ COPY ./addons/owncloud/requirements.txt ./addons/owncloud/ -COPY ./addons/s3/requirements.txt ./addons/s3/ COPY ./addons/twofactor/requirements.txt ./addons/twofactor/ #COPY ./addons/wiki/requirements.txt ./addons/wiki/ COPY ./addons/zotero/requirements.txt ./addons/zotero/ diff --git a/addons/s3/README.md b/addons/s3/README.md index 6b4261bd77a..d3347506daa 100644 --- a/addons/s3/README.md +++ b/addons/s3/README.md @@ -16,3 +16,23 @@ If you already have an access key and ID, skip this step 2. Scroll down to Configure Add-ons 3. Connect your account and enter your ID and secret 4. Select a bucket to work from, or create a new one. + + +## Creating a restricted-access AWS user for S3 connection + +1. Login to AWS +2. Open Identity and Access Management +3. Create a user, assign name, set "Access Type" to "Programmatic Access" +4. Permissions (simple): + 1. Add "AmazonS3FullAccess" policy +5. Permissions (minimal): + 1. Click "Create Policy" => opens a new window + 2. Make a new policy with the following Actions enabled: `["s3:DeleteObject", "s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:ReplicateObject", "s3:RestoreObject", "s3:ListAllMyBuckets", "s3:GetBucketLocation", "s3:CreateBucket"]` + 3. Resources: *I just do "All Resources", but an AWS-educated person would know better about how to narrow it down. + 4. Click "Create" + 5. Return to "Create User" window + 6. Reload policy list + 7. Search for your new policy and select. +6. Tags: *eh* +7. Click "Review", review, then click "Create user" +8. Note "Access key ID" and "Secret access key". These are the inputs to the OSF S3 addon. \ No newline at end of file diff --git a/addons/s3/apps.py b/addons/s3/apps.py index 48943b4d87d..ef520daeed1 100644 --- a/addons/s3/apps.py +++ b/addons/s3/apps.py @@ -12,6 +12,7 @@ class S3AddonAppConfig(BaseAddonAppConfig): + default = True name = 'addons.s3' label = 'addons_s3' full_name = 'Amazon S3' diff --git a/addons/s3/models.py b/addons/s3/models.py index b42a1552696..226467776bb 100644 --- a/addons/s3/models.py +++ b/addons/s3/models.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from addons.base.models import (BaseOAuthNodeSettings, BaseOAuthUserSettings, BaseStorageAddon) from django.db import models @@ -8,11 +6,18 @@ from addons.base import exceptions from addons.s3.provider import S3Provider from addons.s3.serializer import S3Serializer -from addons.s3.settings import (BUCKET_LOCATIONS, - ENCRYPT_UPLOADS_DEFAULT) -from addons.s3.utils import (bucket_exists, - get_bucket_location_or_error, - get_bucket_names) +from addons.s3.settings import ( + BUCKET_LOCATIONS, + ENCRYPT_UPLOADS_DEFAULT +) +from addons.s3.utils import ( + bucket_exists, + get_bucket_location_or_error, + get_bucket_names, + get_bucket_prefixes +) +from website.util import api_v2_url + class S3FileNode(BaseFileNode): _provider = 's3' @@ -53,10 +58,12 @@ def folder_path(self): @property def display_name(self): - return u'{0}: {1}'.format(self.config.full_name, self.folder_id) + return f'{self.config.full_name}: {self.folder_id}' def set_folder(self, folder_id, auth): - if not bucket_exists(self.external_account.oauth_key, self.external_account.oauth_secret, folder_id): + bucket_name = folder_id.split(':')[0] + + if not bucket_exists(self.external_account.oauth_key, self.external_account.oauth_secret, bucket_name): error_message = ('We are having trouble connecting to that bucket. ' 'Try a different one.') raise exceptions.InvalidFolderError(error_message) @@ -66,7 +73,7 @@ def set_folder(self, folder_id, auth): bucket_location = get_bucket_location_or_error( self.external_account.oauth_key, self.external_account.oauth_secret, - folder_id + bucket_name ) try: bucket_location = BUCKET_LOCATIONS[bucket_location] @@ -75,32 +82,46 @@ def set_folder(self, folder_id, auth): # Default to the key. When hit, add mapping to settings pass - self.folder_name = '{} ({})'.format(folder_id, bucket_location) + self.folder_name = f'{folder_id} ({bucket_location})' self.save() - self.nodelogger.log(action='bucket_linked', extra={'bucket': str(folder_id)}, save=True) + self.nodelogger.log(action='bucket_linked', extra={'bucket': bucket_name, 'path': self.folder_id}, save=True) - def get_folders(self, **kwargs): - # This really gets only buckets, not subfolders, - # as that's all we want to be linkable on a node. - try: + def get_folders(self, path, folder_id): + """ + Our S3 implementation allows for folder_id to be a bucket's root, or a subfolder in that bucket. + """ + # This is the root, so list all buckets. + if not folder_id: buckets = get_bucket_names(self) - except Exception: - raise exceptions.InvalidAuthError() - return [ - { + return [{ 'addon': 's3', 'kind': 'folder', - 'id': bucket, + 'id': f'{bucket}:/', 'name': bucket, - 'path': bucket, + 'bucket_name': bucket, + 'path': '/', 'urls': { - 'folders': '' + 'folders': api_v2_url( + f'nodes/{self.owner._id}/addons/s3/folders/', + params={ + 'id': bucket, + 'bucket_name': bucket, + } + ), } - } - for bucket in buckets - ] + } for bucket in buckets] + # This is for a directory for a specific bucket, folders (Prefixes), but not files (Keys) returned, because + # these we can only set folders as our base folder_id + else: + bucket_name, _, path = folder_id.partition(':/') + return get_bucket_prefixes( + self.external_account.oauth_key, + self.external_account.oauth_secret, + prefix=path, + bucket_name=bucket_name + ) @property def complete(self): @@ -124,7 +145,7 @@ def deauthorize(self, auth=None, log=True): def delete(self, save=True): self.deauthorize(log=False) - super(NodeSettings, self).delete(save=save) + super().delete(save=save) def serialize_waterbutler_credentials(self): if not self.has_auth: @@ -135,10 +156,16 @@ def serialize_waterbutler_credentials(self): } def serialize_waterbutler_settings(self): + """ + We use the folder id to hold the bucket location + """ if not self.folder_id: raise exceptions.AddonError('Cannot serialize settings for S3 addon') + + bucket_name = self.folder_id.split(':')[0] return { - 'bucket': self.folder_id, + 'bucket': bucket_name, + 'id': self.folder_id, 'encrypt_uploads': self.encrypt_uploads } @@ -146,7 +173,7 @@ def create_waterbutler_log(self, auth, action, metadata): url = self.owner.web_url_for('addon_view_or_download_file', path=metadata['path'], provider='s3') self.owner.add_log( - 's3_{0}'.format(action), + f's3_{action}', auth=auth, params={ 'project': self.owner.parent_id, diff --git a/addons/s3/provider.py b/addons/s3/provider.py index ba28d065d54..0321a4a7d4e 100644 --- a/addons/s3/provider.py +++ b/addons/s3/provider.py @@ -1,6 +1,6 @@ from addons.s3.serializer import S3Serializer -class S3Provider(object): +class S3Provider: """An alternative to `ExternalProvider` not tied to OAuth""" name = 'Amazon S3' @@ -8,7 +8,7 @@ class S3Provider(object): serializer = S3Serializer def __init__(self, account=None): - super(S3Provider, self).__init__() + super().__init__() # provide an unauthenticated session by default self.account = account diff --git a/addons/s3/requirements.txt b/addons/s3/requirements.txt index 8e6ed87adf9..a61cef6d77f 100644 --- a/addons/s3/requirements.txt +++ b/addons/s3/requirements.txt @@ -1 +1,2 @@ -boto==2.38.0 +# boto (v2) was removed as part of SigV4 migration. +# boto3/botocore are declared in the root requirements.txt. diff --git a/addons/s3/settings/__init__.py b/addons/s3/settings/__init__.py index 2b2f98881f6..40f955c5a78 100644 --- a/addons/s3/settings/__init__.py +++ b/addons/s3/settings/__init__.py @@ -7,4 +7,4 @@ try: from .local import * # noqa except ImportError: - logger.warn('No local.py settings file found') + logger.warning('No local.py settings file found') diff --git a/addons/s3/static/node-cfg.js b/addons/s3/static/node-cfg.js index c7e33ca410a..ccef5f3e251 100644 --- a/addons/s3/static/node-cfg.js +++ b/addons/s3/static/node-cfg.js @@ -4,4 +4,4 @@ var s3NodeConfig = require('./s3NodeConfig.js').s3NodeConfig; var url = window.contextVars.node.urls.api + 's3/settings/'; -new s3NodeConfig('Amazon S3', '#s3Scope', url, '#s3Grid'); +new s3NodeConfig('Amazon S3', '#s3Scope', url, '#s3Grid'); \ No newline at end of file diff --git a/addons/s3/static/s3-rubeus-cfg.js b/addons/s3/static/s3-rubeus-cfg.js index 8493351f9ee..d76dcc8f744 100644 --- a/addons/s3/static/s3-rubeus-cfg.js +++ b/addons/s3/static/s3-rubeus-cfg.js @@ -1,47 +1,46 @@ - var Rubeus = require('rubeus'); +var Rubeus = require('rubeus'); - Rubeus.cfg.s3 = { +Rubeus.cfg.s3 = { - uploadMethod: 'PUT', - uploadUrl: null, - uploadAdded: function(file, item) { - var self = this; - var parent = self.getByID(item.parentID); - var name = file.name; - // Make it possible to upload into subfolders - while (parent.depth > 1 && !parent.isAddonRoot) { - name = parent.name + '/' + name; - parent = self.getByID(parent.parentID); - } - file.destination = name; - file.signedUrlFrom = parent.urls.upload; - }, + uploadMethod: 'PUT', + uploadUrl: null, + uploadAdded: function (file, item) { + var self = this; + var parent = self.getByID(item.parentID); + var name = file.name; + // Make it possible to upload into subfolders + while (parent.depth > 1 && !parent.isAddonRoot) { + name = parent.name + '/' + name; + parent = self.getByID(parent.parentID); + } + file.destination = name; + file.signedUrlFrom = parent.urls.upload; + }, - uploadSending: function(file, formData, xhr) { - xhr.setRequestHeader('Content-Type', file.type || 'application/octet-stream'); - xhr.setRequestHeader('x-amz-acl', 'private'); - }, + uploadSending: function (file, formData, xhr) { + xhr.setRequestHeader('Content-Type', file.type || 'application/octet-stream'); + xhr.setRequestHeader('x-amz-acl', 'private'); + }, - uploadSuccess: function(file, row) { - var self = this; - var parent = this.getByID(row.parentID); - row.urls = { - 'delete': parent.nodeApiUrl + 's3/' + file.destination + '/', - 'download': parent.nodeUrl + 's3/' + file.destination + '/download/', - 'view': parent.nodeUrl + 's3/' + file.destination + '/' - }; - row.permissions = parent.permissions; - this.updateItem(row); - var updated = Rubeus.Utils.itemUpdated(row, parent); - if (updated) { - self.changeStatus(row, Rubeus.Status.UPDATED); - self.delayRemoveRow(row); - } else { - self.changeStatus(row, Rubeus.Status.UPLOAD_SUCCESS, null, 2000, - function(row) { - self.showButtons(row); - }); - } + uploadSuccess: function (file, row) { + var self = this; + var parent = this.getByID(row.parentID); + row.urls = { + 'delete': parent.nodeApiUrl + 's3/' + file.destination + '/', + 'download': parent.nodeUrl + 's3/' + file.destination + '/download/', + 'view': parent.nodeUrl + 's3/' + file.destination + '/' + }; + row.permissions = parent.permissions; + this.updateItem(row); + var updated = Rubeus.Utils.itemUpdated(row, parent); + if (updated) { + self.changeStatus(row, Rubeus.Status.UPDATED); + self.delayRemoveRow(row); + } else { + self.changeStatus(row, Rubeus.Status.UPLOAD_SUCCESS, null, 2000, + function (row) { + self.showButtons(row); + }); } - }; - + } +}; diff --git a/addons/s3/static/s3AnonymousLogActionList.json b/addons/s3/static/s3AnonymousLogActionList.json index 788ddcfa5fd..9f470e89f03 100644 --- a/addons/s3/static/s3AnonymousLogActionList.json +++ b/addons/s3/static/s3AnonymousLogActionList.json @@ -1,11 +1,11 @@ { - "s3_bucket_linked" : "A user linked an Amazon S3 bucket to a project", - "s3_bucket_unlinked" : "A user unselected an Amazon S3 bucket in a project", - "s3_file_added" : "A user added a file to an Amazon S3 bucket in a project", - "s3_file_removed" : "A user removed a file in an Amazon S3 bucket in a project", - "s3_file_updated" : "A user updated a file in an Amazon S3 bucket in a project", - "s3_folder_created" : "A user created a folder in an Amazon S3 in a project", - "s3_node_authorized" : "A user authorized the Amazon S3 addon for a project", - "s3_node_deauthorized" : "A user deauthorized the Amazon S3 addon for a project", - "s3_node_deauthorized_no_user" : "Amazon S3 addon for a project deauthorized" -} + "s3_bucket_linked": "A user linked an Amazon S3 bucket to a project", + "s3_bucket_unlinked": "A user unselected an Amazon S3 bucket in a project", + "s3_file_added": "A user added a file to an Amazon S3 bucket in a project", + "s3_file_removed": "A user removed a file in an Amazon S3 bucket in a project", + "s3_file_updated": "A user updated a file in an Amazon S3 bucket in a project", + "s3_folder_created": "A user created a folder in an Amazon S3 in a project", + "s3_node_authorized": "A user authorized the Amazon S3 addon for a project", + "s3_node_deauthorized": "A user deauthorized the Amazon S3 addon for a project", + "s3_node_deauthorized_no_user": "Amazon S3 addon for a project deauthorized" +} \ No newline at end of file diff --git a/addons/s3/static/s3LogActionList.json b/addons/s3/static/s3LogActionList.json index f2dc7ceef05..63d88ca3ba7 100644 --- a/addons/s3/static/s3LogActionList.json +++ b/addons/s3/static/s3LogActionList.json @@ -1,11 +1,11 @@ { - "s3_bucket_linked" : "${user} linked the Amazon S3 bucket ${bucket} to ${node}", - "s3_bucket_unlinked" : "${user} unselected the Amazon S3 bucket ${bucket} in ${node}", - "s3_file_added" : "${user} added file ${path} to Amazon S3 bucket ${bucket} in ${node}", - "s3_file_removed" : "${user} removed ${path} in Amazon S3 bucket ${bucket} in ${node}", - "s3_file_updated" : "${user} updated file ${path} in Amazon S3 bucket ${bucket} in ${node}", - "s3_folder_created" : "${user} created folder ${path} in Amazon S3 bucket ${bucket} in ${node}", - "s3_node_authorized" : "${user} authorized the Amazon S3 addon for ${node}", - "s3_node_deauthorized" : "${user} deauthorized the Amazon S3 addon for ${node}", - "s3_node_deauthorized_no_user" : "Amazon S3 addon for ${node} deauthorized" -} + "s3_bucket_linked": "${user} linked the Amazon S3 bucket ${bucket} to ${node}", + "s3_bucket_unlinked": "${user} unselected the Amazon S3 bucket ${bucket} in ${node}", + "s3_file_added": "${user} added file ${path} to Amazon S3 bucket ${bucket} in ${node}", + "s3_file_removed": "${user} removed ${path} in Amazon S3 bucket ${bucket} in ${node}", + "s3_file_updated": "${user} updated file ${path} in Amazon S3 bucket ${bucket} in ${node}", + "s3_folder_created": "${user} created folder ${path} in Amazon S3 bucket ${bucket} in ${node}", + "s3_node_authorized": "${user} authorized the Amazon S3 addon for ${node}", + "s3_node_deauthorized": "${user} deauthorized the Amazon S3 addon for ${node}", + "s3_node_deauthorized_no_user": "Amazon S3 addon for ${node} deauthorized" +} \ No newline at end of file diff --git a/addons/s3/static/s3NodeConfig.js b/addons/s3/static/s3NodeConfig.js index f5369691dfb..071deaed584 100644 --- a/addons/s3/static/s3NodeConfig.js +++ b/addons/s3/static/s3NodeConfig.js @@ -15,7 +15,7 @@ var OauthAddonFolderPicker = require('js/oauthAddonNodeConfig')._OauthAddonNodeC var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { bucketLocations: s3Settings.bucketLocations, - constructor: function(addonName, url, selector, folderPicker, opts, tbOpts) { + constructor: function (addonName, url, selector, folderPicker, opts, tbOpts) { var self = this; self.super.constructor(addonName, url, selector, folderPicker, tbOpts); // Non-OAuth fields @@ -26,9 +26,9 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { {}, OauthAddonFolderPicker.prototype.treebeardOptions, { // TreeBeard Options - columnTitles: function() { + columnTitles: function () { return [{ - title: 'Buckets', + title: 'Buckets/Folders', width: '75%', sort: false }, { @@ -37,30 +37,24 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { sort: false }]; }, - resolveToggle: function(item) { - return ''; - }, - resolveIcon: function(item) { - return m('i.fa.fa-folder-o', ' '); - }, }, tbOpts ); }, - connectAccount: function() { + connectAccount: function () { var self = this; - if( !self.accessKey() && !self.secretKey() ){ + if (!self.accessKey() && !self.secretKey()) { self.changeMessage('Please enter both an API access key and secret key.', 'text-danger'); return; } - if (!self.accessKey() ){ + if (!self.accessKey()) { self.changeMessage('Please enter an API access key.', 'text-danger'); return; } - if (!self.secretKey() ){ + if (!self.secretKey()) { self.changeMessage('Please enter an API secret key.', 'text-danger'); return; } @@ -68,17 +62,17 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { return $osf.postJSON( self.urls().create, { - secret_key: self.secretKey(), - access_key: self.accessKey() - } - ).done(function(response) { + secret_key: self.secretKey(), + access_key: self.accessKey() + } + ).done(function (response) { $osf.unblock(); self.clearModal(); $('#s3InputCredentials').modal('hide'); self.changeMessage('Successfully added S3 credentials.', 'text-success', null, true); self.updateFromData(response); self.importAuth(); - }).fail(function(xhr, status, error) { + }).fail(function (xhr, status, error) { $osf.unblock(); var message = ''; var response = JSON.parse(xhr.responseText); @@ -110,7 +104,7 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { * @param {String} bucketName user-provided name of bucket to validate * @param {Boolean} laxChecking whether to use the more permissive validation */ - isValidBucketName: function(bucketName, laxChecking) { + isValidBucketName: function (bucketName, laxChecking) { if (laxChecking === true) { return /^[a-zA-Z0-9.\-_]{1,255}$/.test(bucketName); } @@ -122,7 +116,7 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { }, /** Reset all fields from S3 credentials input modal */ - clearModal: function() { + clearModal: function () { var self = this; self.message(''); self.messageClass('text-info'); @@ -130,22 +124,22 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { self.accessKey(null); }, - createBucket: function(self, bucketName, bucketLocation) { + createBucket: function (self, bucketName, bucketLocation) { $osf.block(); bucketName = bucketName.toLowerCase(); return $osf.postJSON( self.urls().createBucket, { - bucket_name: bucketName, - bucket_location: bucketLocation - } - ).done(function(response) { + bucket_name: bucketName, + bucket_location: bucketLocation + } + ).done(function (response) { $osf.unblock(); self.loadedFolders(false); self.activatePicker(); var msg = 'Successfully created bucket "' + $osf.htmlEscape(bucketName) + '". You can now select it from the list.'; var msgType = 'text-success'; self.changeMessage(msg, msgType, null, true); - }).fail(function(xhr) { + }).fail(function (xhr) { var resp = JSON.parse(xhr.responseText); var message = resp.message; var title = resp.title || 'Problem creating bucket'; @@ -156,21 +150,21 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { bootbox.confirm({ title: $osf.htmlEscape(title), message: $osf.htmlEscape(message), - callback: function(result) { + callback: function (result) { if (result) { self.openCreateBucket(); } }, - buttons:{ - confirm:{ - label:'Try again' + buttons: { + confirm: { + label: 'Try again' } } }); }); }, - openCreateBucket: function() { + openCreateBucket: function () { var self = this; // Generates html options for key-value pairs in BUCKET_LOCATION_MAP @@ -187,32 +181,32 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { bootbox.dialog({ title: 'Create a new bucket', message: - '
      ' + - '
      ' + - '
      ' + - '
      ' + - ' ' + - '
      ' + - ' ' + - '
      ' + - '' + - '
      '+ - '
      ' + - '
      ' + - '
      ' + - ' ' + - '
      ' + - '' + - '
      ' + - '
      ' + - '
      ' + - 'For more information on locations, click ' + - 'here' + - '' + - '
      ' + - '
      ', + '
      ' + + '
      ' + + '
      ' + + '
      ' + + ' ' + + '
      ' + + ' ' + + '
      ' + + '' + + '
      ' + + '
      ' + + '
      ' + + '
      ' + + ' ' + + '
      ' + + '' + + '
      ' + + '
      ' + + '
      ' + + 'See more ' + + 'information on locations' + + '' + + '
      ' + + '
      ', buttons: { cancel: { label: 'Cancel', @@ -234,7 +228,7 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { bootbox.confirm({ title: 'Invalid bucket name', message: 'Amazon S3 buckets can contain lowercase letters, numbers, and hyphens separated by' + - ' periods. Please try another name.', + ' periods. Please try another name.', callback: function (result) { if (result) { self.openCreateBucket(); @@ -271,4 +265,4 @@ function s3NodeConfig(addonName, selector, url, folderPicker, opts, tbOpts) { module.exports = { s3NodeConfig: s3NodeConfig, _s3NodeConfigViewModel: s3FolderPickerViewModel -}; +}; \ No newline at end of file diff --git a/addons/s3/static/s3UserConfig.js b/addons/s3/static/s3UserConfig.js index 5489bc93ca8..ed9c4d433b2 100644 --- a/addons/s3/static/s3UserConfig.js +++ b/addons/s3/static/s3UserConfig.js @@ -32,26 +32,26 @@ function ViewModel(url) { ChangeMessageMixin.call(self); /** Reset all fields from S3 credentials input modal */ - self.clearModal = function() { + self.clearModal = function () { self.message(''); self.messageClass('text-info'); self.accessKey(null); self.secretKey(null); }; /** Send POST request to authorize S3 */ - self.connectAccount = function() { + self.connectAccount = function () { // Selection should not be empty - if( !self.accessKey() && !self.secretKey() ){ + if (!self.accessKey() && !self.secretKey()) { self.changeMessage('Please enter both an API access key and secret key.', 'text-danger'); return; } - if (!self.accessKey() ){ + if (!self.accessKey()) { self.changeMessage('Please enter an API access key.', 'text-danger'); return; } - if (!self.secretKey() ){ + if (!self.secretKey()) { self.changeMessage('Please enter an API secret key.', 'text-danger'); return; } @@ -61,12 +61,12 @@ function ViewModel(url) { access_key: self.accessKey, secret_key: self.secretKey, }) - ).done(function() { + ).done(function () { self.clearModal(); $modal.modal('hide'); self.updateAccounts(); - }).fail(function(xhr, textStatus, error) { + }).fail(function (xhr, textStatus, error) { var errorMessage = (xhr.status === 400 && xhr.responseJSON.message !== undefined) ? xhr.responseJSON.message : language.authError; self.changeMessage(errorMessage, 'text-danger'); Raven.captureMessage('Could not authenticate with S3', { @@ -79,20 +79,20 @@ function ViewModel(url) { }); }; - self.updateAccounts = function() { + self.updateAccounts = function () { return $.ajax({ url: url, type: 'GET', dataType: 'json' }).done(function (data) { - self.accounts($.map(data.accounts, function(account) { - var externalAccount = new ExternalAccount(account); + self.accounts($.map(data.accounts, function (account) { + var externalAccount = new ExternalAccount(account); externalAccount.accessKey = account.oauth_key; externalAccount.secretKey = account.oauth_secret; return externalAccount; })); - $('#s3-header').osfToggleHeight({height: 160}); - }).fail(function(xhr, status, error) { + $('#s3-header').osfToggleHeight({ height: 160 }); + }).fail(function (xhr, status, error) { self.changeMessage(language.userSettingsError, 'text-danger'); Raven.captureMessage('Error while updating addon account', { extra: { @@ -104,7 +104,7 @@ function ViewModel(url) { }); }; - self.askDisconnect = function(account) { + self.askDisconnect = function (account) { var self = this; bootbox.confirm({ title: 'Disconnect Amazon S3 Account?', @@ -117,26 +117,26 @@ function ViewModel(url) { self.disconnectAccount(account); } }, - buttons:{ - confirm:{ - label:'Disconnect', - className:'btn-danger' + buttons: { + confirm: { + label: 'Disconnect', + className: 'btn-danger' } } }); }; - self.disconnectAccount = function(account) { + self.disconnectAccount = function (account) { var self = this; var url = '/api/v1/oauth/accounts/' + account.id + '/'; var request = $.ajax({ url: url, type: 'DELETE' }); - request.done(function(data) { + request.done(function (data) { self.updateAccounts(); }); - request.fail(function(xhr, status, error) { + request.fail(function (xhr, status, error) { Raven.captureMessage('Error while removing addon authorization for ' + account.id, { extra: { url: url, @@ -148,8 +148,8 @@ function ViewModel(url) { return request; }; - self.selectionChanged = function() { - self.changeMessage('',''); + self.selectionChanged = function () { + self.changeMessage('', ''); }; self.updateAccounts(); @@ -170,4 +170,4 @@ function S3UserConfig(selector, url) { module.exports = { S3ViewModel: ViewModel, S3UserConfig: S3UserConfig -}; +}; \ No newline at end of file diff --git a/addons/s3/static/settings.json b/addons/s3/static/settings.json index 05ea9dc29c5..1a5060a1ae4 100644 --- a/addons/s3/static/settings.json +++ b/addons/s3/static/settings.json @@ -16,4 +16,4 @@ "sa-east-1": "Sao Paulo" }, "encryptUploads": true -} +} \ No newline at end of file diff --git a/addons/s3/static/user-cfg.js b/addons/s3/static/user-cfg.js index 48e6c1894e5..474aa68ca73 100644 --- a/addons/s3/static/user-cfg.js +++ b/addons/s3/static/user-cfg.js @@ -3,4 +3,4 @@ var S3UserConfig = require('./s3UserConfig.js').S3UserConfig; // Endpoint for S3 user settings var url = '/api/v1/settings/s3/accounts/'; -var s3UserConfig = new S3UserConfig('#s3AddonScope', url); +var s3UserConfig = new S3UserConfig('#s3AddonScope', url); \ No newline at end of file diff --git a/addons/s3/templates/s3_credentials_modal.mako b/addons/s3/templates/s3_credentials_modal.mako index de0092a9035..f097291f03a 100644 --- a/addons/s3/templates/s3_credentials_modal.mako +++ b/addons/s3/templates/s3_credentials_modal.mako @@ -44,4 +44,4 @@ - + \ No newline at end of file diff --git a/addons/s3/templates/s3_node_settings.mako b/addons/s3/templates/s3_node_settings.mako index 809d86dbb2e..82a127a4e7c 100644 --- a/addons/s3/templates/s3_node_settings.mako +++ b/addons/s3/templates/s3_node_settings.mako @@ -4,7 +4,7 @@ <%include file="s3_credentials_modal.mako"/>

      - + ${addon_full_name} icon ${addon_full_name} @@ -29,7 +29,7 @@

      - + ${_("Connect Account")} @@ -90,4 +90,4 @@

      - + \ No newline at end of file diff --git a/addons/s3/templates/s3_user_settings.mako b/addons/s3/templates/s3_user_settings.mako index dc4274d9319..83f7317d1ab 100644 --- a/addons/s3/templates/s3_user_settings.mako +++ b/addons/s3/templates/s3_user_settings.mako @@ -6,7 +6,7 @@ <%include file="s3_credentials_modal.mako"/>

      - + ${addon_full_name} icon ${_("Connect or Reauthorize Account")} @@ -21,7 +21,7 @@ - + @@ -43,4 +43,4 @@ - + \ No newline at end of file diff --git a/addons/s3/tests/factories.py b/addons/s3/tests/factories.py index a8b49856d47..bafdbaf9c84 100644 --- a/addons/s3/tests/factories.py +++ b/addons/s3/tests/factories.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Factories for the S3 addon.""" import factory from factory.django import DjangoModelFactory @@ -11,9 +10,9 @@ class S3AccountFactory(ExternalAccountFactory): provider = 's3' - provider_id = factory.Sequence(lambda n: 'id-{0}'.format(n)) - oauth_key = factory.Sequence(lambda n: 'key-{0}'.format(n)) - oauth_secret = factory.Sequence(lambda n: 'secret-{0}'.format(n)) + provider_id = factory.Sequence(lambda n: f'id-{n}') + oauth_key = factory.Sequence(lambda n: f'key-{n}') + oauth_secret = factory.Sequence(lambda n: f'secret-{n}') display_name = 'S3 Fake User' diff --git a/addons/s3/tests/test_model.py b/addons/s3/tests/test_model.py index b616a66c652..af608f5cc8a 100644 --- a/addons/s3/tests/test_model.py +++ b/addons/s3/tests/test_model.py @@ -1,7 +1,4 @@ -# from nose.tools import * # noqa -import mock -from nose.tools import (assert_false, assert_true, - assert_equal, assert_is_none) +from unittest import mock import pytest import unittest @@ -38,26 +35,33 @@ class TestNodeSettings(OAuthAddonNodeSettingsTestSuiteMixin, unittest.TestCase): NodeSettingsClass = NodeSettings UserSettingsFactory = S3UserSettingsFactory + def _node_settings_class_kwargs(self, node, user_settings): + return { + 'user_settings': self.user_settings, + 'folder_id': 'bucket_name:/path_goes_here/with_folder_id', + 'owner': self.node + } + def test_registration_settings(self): registration = ProjectFactory() clone, message = self.node_settings.after_register( self.node, registration, self.user, ) - assert_is_none(clone) + assert clone is None def test_before_register_no_settings(self): self.node_settings.user_settings = None message = self.node_settings.before_register(self.node, self.user) - assert_false(message) + assert not message def test_before_register_no_auth(self): self.node_settings.external_account = None message = self.node_settings.before_register(self.node, self.user) - assert_false(message) + assert not message def test_before_register_settings_and_auth(self): message = self.node_settings.before_register(self.node, self.user) - assert_true(message) + assert message @mock.patch('website.archiver.tasks.archive') def test_does_not_get_copied_to_registrations(self, mock_archive): @@ -66,7 +70,7 @@ def test_does_not_get_copied_to_registrations(self, mock_archive): auth=Auth(user=self.user), draft_registration=DraftRegistrationFactory(branched_from=self.node), ) - assert_false(registration.has_addon('s3')) + assert not registration.has_addon('s3') ## Overrides ## @@ -78,7 +82,7 @@ def test_serialize_credentials(self): expected = {'access_key': self.node_settings.external_account.oauth_key, 'secret_key': self.node_settings.external_account.oauth_secret} - assert_equal(credentials, expected) + assert credentials == expected @mock.patch('addons.s3.models.bucket_exists') @mock.patch('addons.s3.models.get_bucket_location_or_error') @@ -89,13 +93,16 @@ def test_set_folder(self, mock_location, mock_exists): self.node_settings.set_folder(folder_id, auth=Auth(self.user)) self.node_settings.save() # Bucket was set - assert_equal(self.node_settings.folder_id, folder_id) + assert self.node_settings.folder_id == folder_id # Log was saved last_log = self.node.logs.latest() - assert_equal(last_log.action, '{0}_bucket_linked'.format(self.short_name)) + assert last_log.action == f'{self.short_name}_bucket_linked' def test_serialize_settings(self): settings = self.node_settings.serialize_waterbutler_settings() - expected = {'bucket': self.node_settings.folder_id, - 'encrypt_uploads': self.node_settings.encrypt_uploads} - assert_equal(settings, expected) + expected = { + 'bucket': 'bucket_name', + 'encrypt_uploads': self.node_settings.encrypt_uploads, + 'id': 'bucket_name:/path_goes_here/with_folder_id' + } + assert settings == expected diff --git a/addons/s3/tests/test_serializer.py b/addons/s3/tests/test_serializer.py index a1c637020c3..f1fa025d49c 100644 --- a/addons/s3/tests/test_serializer.py +++ b/addons/s3/tests/test_serializer.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- """Serializer tests for the S3 addon.""" -import mock +from unittest import mock import pytest from addons.base.tests.serializers import StorageAddonSerializerTestSuiteMixin @@ -24,8 +23,8 @@ def setUp(self): self.mock_can_list = mock.patch('addons.s3.serializer.utils.can_list') self.mock_can_list.return_value = True self.mock_can_list.start() - super(TestS3Serializer, self).setUp() + super().setUp() def tearDown(self): self.mock_can_list.stop() - super(TestS3Serializer, self).tearDown() + super().tearDown() diff --git a/addons/s3/tests/test_view.py b/addons/s3/tests/test_view.py index 8db2b2a9091..47b0ad9435d 100644 --- a/addons/s3/tests/test_view.py +++ b/addons/s3/tests/test_view.py @@ -1,15 +1,12 @@ -# -*- coding: utf-8 -*- from rest_framework import status as http_status -from boto.exception import S3ResponseError -import mock -from nose.tools import (assert_equal, assert_equals, - assert_true, assert_in, assert_false) +from botocore.exceptions import NoCredentialsError +from unittest import mock import pytest from framework.auth import Auth from tests.base import OsfTestCase, get_default_metaschema -from osf_tests.factories import ProjectFactory, AuthUserFactory, DraftRegistrationFactory, InstitutionFactory +from osf_tests.factories import ProjectFactory, AuthUserFactory, DraftRegistrationFactory from addons.base.tests.views import ( OAuthAddonConfigViewsTestCaseMixin @@ -32,35 +29,35 @@ def setUp(self): self.mock_exists = mock.patch('addons.s3.views.utils.bucket_exists') self.mock_exists.return_value = True self.mock_exists.start() - super(TestS3Views, self).setUp() + super().setUp() def tearDown(self): self.mock_can_list.stop() self.mock_uid.stop() self.mock_exists.stop() - super(TestS3Views, self).tearDown() + super().tearDown() def test_s3_settings_input_empty_keys(self): url = self.project.api_url_for('s3_add_user_account') - rv = self.app.post_json(url, { + rv = self.app.post(url, json={ 'access_key': '', 'secret_key': '' - }, auth=self.user.auth, expect_errors=True) - assert_equals(rv.status_int, http_status.HTTP_400_BAD_REQUEST) - assert_in('All the fields above are required.', rv.body.decode()) + }, auth=self.user.auth, ) + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST + assert 'All the fields above are required.' in rv.text def test_s3_settings_input_empty_access_key(self): url = self.project.api_url_for('s3_add_user_account') - rv = self.app.post_json(url, { + rv = self.app.post(url, json={ 'access_key': '', 'secret_key': 'Non-empty-secret-key' - }, auth=self.user.auth, expect_errors=True) - assert_equals(rv.status_int, http_status.HTTP_400_BAD_REQUEST) - assert_in('All the fields above are required.', rv.body.decode()) + }, auth=self.user.auth, ) + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST + assert 'All the fields above are required.' in rv.text def test_s3_settings_input_empty_secret_key(self): url = self.project.api_url_for('s3_add_user_account') - rv = self.app.post_json(url, { + rv = self.app.post(url, json={ 'access_key': 'Non-empty-access-key', 'secret_key': '' }, auth=self.user.auth, expect_errors=True) @@ -86,11 +83,10 @@ def test_s3_set_bucket_no_settings(self): user = AuthUserFactory() self.project.add_contributor(user, save=True) url = self.project.api_url_for('s3_set_config') - res = self.app.put_json( - url, {'s3_bucket': 'hammertofall'}, auth=user.auth, - expect_errors=True + res = self.app.put( + url, json={'s3_bucket': 'hammertofall'}, auth=user.auth, ) - assert_equal(res.status_code, http_status.HTTP_400_BAD_REQUEST) + assert res.status_code == http_status.HTTP_400_BAD_REQUEST def test_s3_set_bucket_no_auth(self): @@ -98,11 +94,10 @@ def test_s3_set_bucket_no_auth(self): user.add_addon('s3') self.project.add_contributor(user, save=True) url = self.project.api_url_for('s3_set_config') - res = self.app.put_json( - url, {'s3_bucket': 'hammertofall'}, auth=user.auth, - expect_errors=True + res = self.app.put( + url, json={'s3_bucket': 'hammertofall'}, auth=user.auth, ) - assert_equal(res.status_code, http_status.HTTP_403_FORBIDDEN) + assert res.status_code == http_status.HTTP_403_FORBIDDEN def test_s3_set_bucket_registered(self): registration = self.project.register_node( @@ -110,35 +105,34 @@ def test_s3_set_bucket_registered(self): ) url = registration.api_url_for('s3_set_config') - res = self.app.put_json( - url, {'s3_bucket': 'hammertofall'}, auth=self.user.auth, - expect_errors=True, + res = self.app.put( + url, json={'s3_bucket': 'hammertofall'}, auth=self.user.auth, ) - assert_equal(res.status_code, http_status.HTTP_400_BAD_REQUEST) + assert res.status_code == http_status.HTTP_400_BAD_REQUEST @mock.patch('addons.s3.views.utils.can_list', return_value=False) def test_user_settings_cant_list(self, mock_can_list): url = api_url_for('s3_add_user_account') - rv = self.app.post_json(url, { + rv = self.app.post(url, json={ 'access_key': 'aldkjf', 'secret_key': 'las' - }, auth=self.user.auth, expect_errors=True) + }, auth=self.user.auth) - assert_in('Unable to list buckets.', rv.body.decode()) - assert_equals(rv.status_int, http_status.HTTP_400_BAD_REQUEST) + assert 'Unable to list buckets.' in rv.text + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST def test_s3_remove_node_settings_owner(self): url = self.node_settings.owner.api_url_for('s3_deauthorize_node') self.app.delete(url, auth=self.user.auth) result = self.Serializer().serialize_settings(node_settings=self.node_settings, current_user=self.user) - assert_equal(result['nodeHasAuth'], False) + assert result['nodeHasAuth'] == False def test_s3_remove_node_settings_unauthorized(self): url = self.node_settings.owner.api_url_for('s3_deauthorize_node') - ret = self.app.delete(url, auth=None, expect_errors=True) + ret = self.app.delete(url, auth=None, ) - assert_equal(ret.status_code, 401) + assert ret.status_code == 401 def test_s3_get_node_settings_owner(self): self.node_settings.set_auth(self.external_account, self.user) @@ -148,23 +142,23 @@ def test_s3_get_node_settings_owner(self): res = self.app.get(url, auth=self.user.auth) result = res.json['result'] - assert_equal(result['nodeHasAuth'], True) - assert_equal(result['userIsOwner'], True) - assert_equal(result['folder']['path'], self.node_settings.folder_id) + assert result['nodeHasAuth'] == True + assert result['userIsOwner'] == True + assert result['folder']['path'] == self.node_settings.folder_id def test_s3_get_node_settings_unauthorized(self): url = self.node_settings.owner.api_url_for('s3_get_config') unauthorized = AuthUserFactory() - ret = self.app.get(url, auth=unauthorized.auth, expect_errors=True) + ret = self.app.get(url, auth=unauthorized.auth, ) - assert_equal(ret.status_code, 403) + assert ret.status_code == 403 ## Overrides ## @mock.patch('addons.s3.models.get_bucket_names') def test_folder_list(self, mock_names): mock_names.return_value = ['bucket1', 'bucket2'] - super(TestS3Views, self).test_folder_list() + super().test_folder_list() @mock.patch('addons.s3.models.bucket_exists') @mock.patch('addons.s3.models.get_bucket_location_or_error') @@ -172,25 +166,22 @@ def test_set_config(self, mock_location, mock_exists): mock_exists.return_value = True mock_location.return_value = '' self.node_settings.set_auth(self.external_account, self.user) - url = self.project.api_url_for('{0}_set_config'.format(self.ADDON_SHORT_NAME)) - res = self.app.put_json(url, { + url = self.project.api_url_for(f'{self.ADDON_SHORT_NAME}_set_config') + res = self.app.put(url, json={ 'selected': self.folder }, auth=self.user.auth) - assert_equal(res.status_code, http_status.HTTP_200_OK) + assert res.status_code == http_status.HTTP_200_OK self.project.reload() self.node_settings.reload() - assert_equal( - self.project.logs.latest().action, - '{0}_bucket_linked'.format(self.ADDON_SHORT_NAME) - ) - assert_equal(res.json['result']['folder']['name'], self.node_settings.folder_name) + assert self.project.logs.latest().action == f'{self.ADDON_SHORT_NAME}_bucket_linked' + assert res.json['result']['folder']['name'] == self.node_settings.folder_name class TestCreateBucket(S3AddonTestCase, OsfTestCase): def setUp(self): - super(TestCreateBucket, self).setUp() + super().setUp() self.user = AuthUserFactory() self.consolidated_auth = Auth(user=self.user) @@ -212,57 +203,57 @@ def setUp(self): self.node_settings.save() def test_bad_names(self): - assert_false(validate_bucket_name('')) - assert_false(validate_bucket_name('no')) - assert_false(validate_bucket_name('a' * 64)) - assert_false(validate_bucket_name(' leadingspace')) - assert_false(validate_bucket_name('trailingspace ')) - assert_false(validate_bucket_name('bogus naMe')) - assert_false(validate_bucket_name('.cantstartwithp')) - assert_false(validate_bucket_name('or.endwith.')) - assert_false(validate_bucket_name('..nodoubles')) - assert_false(validate_bucket_name('no_unders_in')) - assert_false(validate_bucket_name('-leadinghyphen')) - assert_false(validate_bucket_name('trailinghyphen-')) - assert_false(validate_bucket_name('Mixedcase')) - assert_false(validate_bucket_name('empty..label')) - assert_false(validate_bucket_name('label-.trailinghyphen')) - assert_false(validate_bucket_name('label.-leadinghyphen')) - assert_false(validate_bucket_name('8.8.8.8')) - assert_false(validate_bucket_name('600.9000.0.28')) - assert_false(validate_bucket_name('no_underscore')) - assert_false(validate_bucket_name('_nounderscoreinfront')) - assert_false(validate_bucket_name('no-underscore-in-back_')) - assert_false(validate_bucket_name('no-underscore-in_the_middle_either')) + assert not validate_bucket_name('') + assert not validate_bucket_name('no') + assert not validate_bucket_name('a' * 64) + assert not validate_bucket_name(' leadingspace') + assert not validate_bucket_name('trailingspace ') + assert not validate_bucket_name('bogus naMe') + assert not validate_bucket_name('.cantstartwithp') + assert not validate_bucket_name('or.endwith.') + assert not validate_bucket_name('..nodoubles') + assert not validate_bucket_name('no_unders_in') + assert not validate_bucket_name('-leadinghyphen') + assert not validate_bucket_name('trailinghyphen-') + assert not validate_bucket_name('Mixedcase') + assert not validate_bucket_name('empty..label') + assert not validate_bucket_name('label-.trailinghyphen') + assert not validate_bucket_name('label.-leadinghyphen') + assert not validate_bucket_name('8.8.8.8') + assert not validate_bucket_name('600.9000.0.28') + assert not validate_bucket_name('no_underscore') + assert not validate_bucket_name('_nounderscoreinfront') + assert not validate_bucket_name('no-underscore-in-back_') + assert not validate_bucket_name('no-underscore-in_the_middle_either') def test_names(self): - assert_true(validate_bucket_name('imagoodname')) - assert_true(validate_bucket_name('still.passing')) - assert_true(validate_bucket_name('can-have-dashes')) - assert_true(validate_bucket_name('kinda.name.spaced')) - assert_true(validate_bucket_name('a-o.valid')) - assert_true(validate_bucket_name('11.12.m')) - assert_true(validate_bucket_name('a--------a')) - assert_true(validate_bucket_name('a' * 63)) + assert validate_bucket_name('imagoodname') + assert validate_bucket_name('still.passing') + assert validate_bucket_name('can-have-dashes') + assert validate_bucket_name('kinda.name.spaced') + assert validate_bucket_name('a-o.valid') + assert validate_bucket_name('11.12.m') + assert validate_bucket_name('a--------a') + assert validate_bucket_name('a' * 63) def test_bad_locations(self): - assert_false(validate_bucket_location('Venus')) - assert_false(validate_bucket_location('AlphaCentari')) - assert_false(validate_bucket_location('CostaRica')) + assert not validate_bucket_location('Venus') + assert not validate_bucket_location('AlphaCentauri') + assert not validate_bucket_location('CostaRica') def test_locations(self): - assert_true(validate_bucket_location('')) - assert_true(validate_bucket_location('eu-central-1')) - assert_true(validate_bucket_location('ca-central-1')) - assert_true(validate_bucket_location('us-west-1')) - assert_true(validate_bucket_location('us-west-2')) - assert_true(validate_bucket_location('ap-northeast-1')) - assert_true(validate_bucket_location('ap-northeast-2')) - assert_true(validate_bucket_location('ap-southeast-1')) - assert_true(validate_bucket_location('ap-southeast-2')) - assert_true(validate_bucket_location('sa-east-1')) - assert_true(validate_bucket_location('eu-west-1')) - assert_true(validate_bucket_location('eu-west-2')) + assert validate_bucket_location('') + assert validate_bucket_location('eu-central-1') + assert validate_bucket_location('ca-central-1') + assert validate_bucket_location('us-west-1') + assert validate_bucket_location('us-west-2') + assert validate_bucket_location('ap-northeast-1') + assert validate_bucket_location('ap-northeast-2') + assert validate_bucket_location('ap-southeast-1') + assert validate_bucket_location('ap-southeast-2') + assert validate_bucket_location('sa-east-1') + assert validate_bucket_location('eu-west-1') + assert validate_bucket_location('eu-west-2') @mock.patch('addons.s3.views.utils.create_bucket') @mock.patch('addons.s3.views.utils.get_bucket_names') @@ -274,39 +265,37 @@ def test_create_bucket_pass(self, mock_names, mock_make): 'doesntevenmatter' ] url = self.project.api_url_for('create_bucket') - ret = self.app.post_json( + ret = self.app.post( url, - { + json={ 'bucket_name': 'doesntevenmatter', 'bucket_location': '', }, auth=self.user.auth ) - assert_equal(ret.status_int, http_status.HTTP_200_OK) - assert_equal(ret.json, {}) + assert ret.status_code == http_status.HTTP_200_OK + assert ret.json == {} @mock.patch('addons.s3.views.utils.create_bucket') def test_create_bucket_fail(self, mock_make): - error = S3ResponseError(418, 'because Im a test') - error.message = 'This should work' + error = NoCredentialsError(operation_name='create_bucket') mock_make.side_effect = error - url = '/api/v1/project/{0}/s3/newbucket/'.format(self.project._id) - ret = self.app.post_json(url, {'bucket_name': 'doesntevenmatter'}, auth=self.user.auth, expect_errors=True) + url = f'/api/v1/project/{self.project._id}/s3/newbucket/' + ret = self.app.post(url, json={'bucket_name': 'doesntevenmatter'}, auth=self.user.auth) - assert_equals(ret.body.decode(), '{"message": "This should work", "title": "Problem connecting to S3"}') + assert ret.text == '{"message": "Unable to locate credentials", "title": "Problem connecting to S3"}' @mock.patch('addons.s3.views.utils.create_bucket') def test_bad_location_fails(self, mock_make): - url = '/api/v1/project/{0}/s3/newbucket/'.format(self.project._id) - ret = self.app.post_json( + url = f'/api/v1/project/{self.project._id}/s3/newbucket/' + ret = self.app.post( url, - { + json={ 'bucket_name': 'doesntevenmatter', 'bucket_location': 'not a real bucket location', }, auth=self.user.auth, - expect_errors=True) - - assert_equals(ret.body.decode(), '{"message": "That bucket location is not valid.", "title": "Invalid bucket location"}') + ) + assert ret.text == '{"message": "That bucket location is not valid.", "title": "Invalid bucket location"}' diff --git a/addons/s3/tests/utils.py b/addons/s3/tests/utils.py index 15b0153f23b..e83f4ebf489 100644 --- a/addons/s3/tests/utils.py +++ b/addons/s3/tests/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from addons.base.tests.base import OAuthAddonTestCaseMixin, AddonTestCase from addons.s3.tests.factories import S3AccountFactory from addons.s3.provider import S3Provider diff --git a/addons/s3/utils.py b/addons/s3/utils.py index 0738b5bf397..73d53ebf18e 100644 --- a/addons/s3/utils.py +++ b/addons/s3/utils.py @@ -1,14 +1,19 @@ import re +import logging +from dataclasses import dataclass +from typing import Optional # Python 3.6 does not support | operator for type hints + from rest_framework import status as http_status -from boto import exception -from boto.s3.connection import S3Connection -from boto.s3.connection import OrdinaryCallingFormat +import boto3 +from botocore import exceptions from framework.exceptions import HTTPError from addons.base.exceptions import InvalidAuthError, InvalidFolderError from addons.s3.settings import BUCKET_LOCATIONS +logger = logging.getLogger(__name__) + def connect_s3(access_key=None, secret_key=None, node_settings=None): """Helper to build an S3Connection object @@ -18,19 +23,27 @@ def connect_s3(access_key=None, secret_key=None, node_settings=None): if node_settings is not None: if node_settings.external_account is not None: access_key, secret_key = node_settings.external_account.oauth_key, node_settings.external_account.oauth_secret - connection = S3Connection(access_key, secret_key, calling_format=OrdinaryCallingFormat()) + connection = boto3.client( + 's3', + aws_access_key_id=access_key, + aws_secret_access_key=secret_key + ) return connection +def get_status_for_error(e: exceptions.ClientError) -> int: + return e.response['ResponseMetadata']['HTTPStatusCode'] + + def get_bucket_names(node_settings): try: - buckets = connect_s3(node_settings=node_settings).get_all_buckets() - except exception.NoAuthHandlerFound: + response = connect_s3(node_settings=node_settings).list_buckets() + except exceptions.NoCredentialsError: raise HTTPError(http_status.HTTP_403_FORBIDDEN) - except exception.BotoServerError as e: - raise HTTPError(e.status) + except exceptions.ClientError as e: + raise HTTPError(get_status_for_error(e)) - return [bucket.name for bucket in buckets] + return [bucket['Name'] for bucket in response['Buckets']] def validate_bucket_location(location): @@ -51,28 +64,34 @@ def validate_bucket_name(name): def create_bucket(node_settings, bucket_name, location=''): - return connect_s3(node_settings=node_settings).create_bucket(bucket_name, location=location) + client = connect_s3(node_settings=node_settings) + + # default bucket location won't work with location constraint + if not location or location == 'us-east-1': + return client.create_bucket(Bucket=bucket_name) + else: + return client.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration={ + 'LocationConstraint': location, + } + ) def bucket_exists(access_key, secret_key, bucket_name): - """Tests for the existance of a bucket and if the user + """Tests for the existence of a bucket and if the user can access it with the given keys """ if not bucket_name: return False - - connection = connect_s3(access_key, secret_key) - - if bucket_name != bucket_name.lower(): - # Must use ordinary calling format for mIxEdCaSe bucket names - # otherwise use the default as it handles bucket outside of the US - connection.calling_format = OrdinaryCallingFormat() + # bucket names are dns-compliant, therefore must be lowercase + bucket_name = bucket_name.lower() try: # Will raise an exception if bucket_name doesn't exist - connect_s3(access_key, secret_key).head_bucket(bucket_name) - except exception.S3ResponseError as e: - if e.status not in (301, 302): + connect_s3(access_key, secret_key).head_bucket(Bucket=bucket_name) + except exceptions.ClientError as e: + if get_status_for_error(e) not in (301, 302): return False return True @@ -87,38 +106,73 @@ def can_list(access_key, secret_key): return False try: - connect_s3(access_key, secret_key).get_all_buckets() - except exception.S3ResponseError: + connect_s3(access_key, secret_key).list_buckets() + except exceptions.ClientError: return False return True -def get_user_info(access_key, secret_key): + +@dataclass(frozen=True) # slots is not supported in python 3.6 +class Owner: + __slots__ = ['display_name', 'id'] + + display_name: str + id: str + + @classmethod + def from_dict(cls, data): + return cls(data['DisplayName'], data['ID']) + + +def get_user_info(access_key: str, secret_key: str) -> Optional[Owner]: """Returns an S3 User with .display_name and .id, or None """ if not (access_key and secret_key): return None try: - return connect_s3(access_key, secret_key).get_all_buckets().owner - except exception.S3ResponseError: + return Owner.from_dict(connect_s3(access_key, secret_key).list_buckets()['Owner']) + except exceptions.ClientError: return None - return None + def get_bucket_location_or_error(access_key, secret_key, bucket_name): """Returns the location of a bucket or raises AddonError """ - try: - connection = connect_s3(access_key, secret_key) - except Exception: - raise InvalidAuthError() - - if bucket_name != bucket_name.lower() or '.' in bucket_name: - # Must use ordinary calling format for mIxEdCaSe bucket names - # otherwise use the default as it handles bucket outside of the US - connection.calling_format = OrdinaryCallingFormat() + # bucket names are dns-compliant, therefore must be lowercase + bucket_name = bucket_name.lower() try: # Will raise an exception if bucket_name doesn't exist - return connect_s3(access_key, secret_key).get_bucket(bucket_name, validate=False).get_location() - except exception.S3ResponseError: + return connect_s3(access_key, secret_key).get_bucket_location(Bucket=bucket_name)['LocationConstraint'] + except exceptions.NoCredentialsError: + raise InvalidAuthError() + except exceptions.ClientError: raise InvalidFolderError() + + +def get_bucket_prefixes(access_key, secret_key, prefix, bucket_name): + s3 = boto3.client( + 's3', + aws_access_key_id=access_key, + aws_secret_access_key=secret_key + ) + + result = s3.list_objects(Bucket=bucket_name, Prefix=prefix, Delimiter='/') + folders = [] + for common_prefixes in result.get('CommonPrefixes', []): + key_name = common_prefixes.get('Prefix') + if key_name != prefix: + folders.append( + { + 'path': key_name, + 'id': f'{bucket_name}:/{key_name}', + 'folder_id': key_name, + 'kind': 'folder', + 'bucket_name': bucket_name, + 'name': key_name.split('/')[-2], + 'addon': 's3', + } + ) + + return folders diff --git a/addons/s3/views.py b/addons/s3/views.py index 672dd335305..5aef084cd90 100644 --- a/addons/s3/views.py +++ b/addons/s3/views.py @@ -1,6 +1,6 @@ from rest_framework import status as http_status -from boto import exception +from botocore import exceptions from django.core.exceptions import ValidationError from flask import request @@ -58,7 +58,9 @@ def _set_folder(node_addon, folder, auth): def s3_folder_list(node_addon, **kwargs): """ Returns all the subsequent folders under the folder id passed. """ - return node_addon.get_folders() + path = request.args.get('path', '') + folder_id = request.args.get('id', '') + return node_addon.get_folders(path=path, folder_id=folder_id) @must_be_logged_in @must_be_rdm_addons_allowed(SHORT_NAME) @@ -148,20 +150,15 @@ def create_bucket(auth, node_addon, **kwargs): try: utils.create_bucket(node_addon, bucket_name, bucket_location) - except exception.S3ResponseError as e: + except exceptions.NoCredentialsError as e: return { - 'message': e.message, + 'message': str(e), 'title': 'Problem connecting to S3', }, http_status.HTTP_400_BAD_REQUEST - except exception.S3CreateError as e: + except exceptions.ClientError as e: return { - 'message': e.message, - 'title': "Problem creating bucket '{0}'".format(bucket_name), - }, http_status.HTTP_400_BAD_REQUEST - except exception.BotoClientError as e: # Base class catchall - return { - 'message': e.message, - 'title': 'Error connecting to S3', + 'message': str(e), + 'title': f"Problem creating bucket '{bucket_name}'", }, http_status.HTTP_400_BAD_REQUEST return {} diff --git a/requirements.txt b/requirements.txt index c9ee19ae23b..f3574223512 100644 --- a/requirements.txt +++ b/requirements.txt @@ -133,3 +133,5 @@ Babel==2.5.1 Flask-Babel==1.0.0 rsa==3.1.4 flask-compress==1.15 + +dataclasses==0.8 \ No newline at end of file From 2f451dc584a40daf8ebed39e826e90f37134d283 Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Fri, 2 Oct 2026 15:27:41 +0900 Subject: [PATCH 23/41] fix(s3): reuse the existing external account when the access key is already registered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registering the same access key twice has to end up on the existing ExternalAccount instead of creating a second row. The duplicate is caught by the unique constraint on (provider, provider_id) — provider_id holds "{aws_account_id}\t{access_key}" (GRDM-53044) — so the save raises IntegrityError, not the ValidationError the handler was catching, and the failed INSERT aborted the surrounding transaction. Wrap the save in transaction.atomic() so the failure only rolls back to a savepoint, and catch IntegrityError alongside ValidationError before falling back to the account that is already stored. --- addons/s3/views.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/addons/s3/views.py b/addons/s3/views.py index 5aef084cd90..1627bcf1a7f 100644 --- a/addons/s3/views.py +++ b/addons/s3/views.py @@ -2,6 +2,7 @@ from botocore import exceptions from django.core.exceptions import ValidationError +from django.db import IntegrityError, transaction from flask import request from framework.exceptions import HTTPError @@ -97,16 +98,17 @@ def s3_add_user_account(auth, **kwargs): masked_access_key = f'****{access_key[-4:]}' if len(access_key) > 4 else '****' display_name = f'{user_info.display_name} ({masked_access_key})' try: - account = ExternalAccount( - provider=SHORT_NAME, - provider_name=FULL_NAME, - oauth_key=access_key, - oauth_secret=secret_key, - provider_id=provider_id, - display_name=display_name, - ) - account.save() - except ValidationError: + with transaction.atomic(): + account = ExternalAccount( + provider=SHORT_NAME, + provider_name=FULL_NAME, + oauth_key=access_key, + oauth_secret=secret_key, + provider_id=provider_id, + display_name=display_name, + ) + account.save() + except (ValidationError, IntegrityError): # ... or get the old one account = ExternalAccount.objects.get( provider=SHORT_NAME, From f70e7f0015e757588a614b63f1084709308a4cdd Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Fri, 2 Oct 2026 15:28:16 +0900 Subject: [PATCH 24/41] test(s3): drop nose remnants and update the node addons mock to the SigV4 folder shape - test_view.py still used assert_equals/assert_in and a missing InstitutionFactory import left over from the nose era; replace them with plain pytest asserts against rv.text - _mock_folder_result still returned the pre-SigV4 get_folders payload, so the node settings tests asserted against a shape the view no longer produces --- addons/s3/tests/test_view.py | 12 ++++++------ api_tests/nodes/views/test_node_addons.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/addons/s3/tests/test_view.py b/addons/s3/tests/test_view.py index 47b0ad9435d..56dd8c26f69 100644 --- a/addons/s3/tests/test_view.py +++ b/addons/s3/tests/test_view.py @@ -6,7 +6,7 @@ from framework.auth import Auth from tests.base import OsfTestCase, get_default_metaschema -from osf_tests.factories import ProjectFactory, AuthUserFactory, DraftRegistrationFactory +from osf_tests.factories import ProjectFactory, AuthUserFactory, DraftRegistrationFactory, InstitutionFactory from addons.base.tests.views import ( OAuthAddonConfigViewsTestCaseMixin @@ -61,8 +61,8 @@ def test_s3_settings_input_empty_secret_key(self): 'access_key': 'Non-empty-access-key', 'secret_key': '' }, auth=self.user.auth, expect_errors=True) - assert_equals(rv.status_int, http_status.HTTP_400_BAD_REQUEST) - assert_in('All the fields above are required.', rv.body.decode()) + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST + assert 'All the fields above are required.' in rv.text def test_s3_settings_rdm_addons_denied(self): institution = InstitutionFactory() @@ -72,12 +72,12 @@ def test_s3_settings_rdm_addons_denied(self): rdm_addon_option.is_allowed = False rdm_addon_option.save() url = self.project.api_url_for('s3_add_user_account') - rv = self.app.post_json(url,{ + rv = self.app.post(url, json={ 'access_key': 'aldkjf', 'secret_key': 'las' }, auth=self.user.auth, expect_errors=True) - assert_equal(rv.status_int, http_status.HTTP_403_FORBIDDEN) - assert_in(b'You are prohibited from using this add-on.', rv.body) + assert rv.status_code == http_status.HTTP_403_FORBIDDEN + assert 'You are prohibited from using this add-on.' in rv.text def test_s3_set_bucket_no_settings(self): user = AuthUserFactory() diff --git a/api_tests/nodes/views/test_node_addons.py b/api_tests/nodes/views/test_node_addons.py index 459731af87c..660da131d84 100644 --- a/api_tests/nodes/views/test_node_addons.py +++ b/api_tests/nodes/views/test_node_addons.py @@ -1015,8 +1015,8 @@ def _settings_kwargs(self, node, user_settings): def _mock_folder_result(self): return { 'name': 'a.bucket', - 'path': 'a.bucket', - 'id': 'a.bucket' + 'path': '/', + 'id': 'a.bucket:/' } @mock.patch('addons.s3.models.get_bucket_names') From 578a9ebf3b4f0f63129f85b93d8779446c5beca5 Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Fri, 2 Oct 2026 15:29:31 +0900 Subject: [PATCH 25/41] test(s3): cover folder_id formats, provider_id dedup and error paths Extend addons/s3/tests/test_view.py so the SigV4 flow is exercised end to end: - both folder_id forms ("bucket" and "bucket:/prefix/") and a non-existent bucket raising InvalidFolderError - the provider_id format "{aws_account_id}\t{access_key}" and the dedup path when the same key is registered twice (GRDM-53044) - the error branches: a missing key field, get_user_info returning None, an invalid bucket name, an empty bucket_exists result, can_list returning None, and the ClientError / InvalidAuthError conversions Three test-harness fixes come with it: - query ExternalAccount by provider_id, a plain CharField. oauth_key is an EncryptedTextField (JWE), so its ciphertext differs on every save and filtering on it never matches; the plaintext is checked by attribute access instead - use post_json/put_json instead of the json= keyword, which the webtest_plus version used in CI does not accept, and pass expect_errors=True wherever a 4xx/401/403 response is expected - patch get_user_info with mock.patch(..., return_value=mock.Mock(...)); the .return_value attribute on the patcher object never reaches the started mock, so user_info.id / user_info.display_name were missing in the view --- addons/s3/tests/test_model.py | 16 ++++ addons/s3/tests/test_view.py | 159 +++++++++++++++++++++++++++++----- 2 files changed, 151 insertions(+), 24 deletions(-) diff --git a/addons/s3/tests/test_model.py b/addons/s3/tests/test_model.py index af608f5cc8a..ac148c59812 100644 --- a/addons/s3/tests/test_model.py +++ b/addons/s3/tests/test_model.py @@ -98,6 +98,22 @@ def test_set_folder(self, mock_location, mock_exists): last_log = self.node.logs.latest() assert last_log.action == f'{self.short_name}_bucket_linked' + @mock.patch('addons.s3.models.bucket_exists') + def test_set_folder_nonexistent_bucket(self, mock_exists): + from addons.base import exceptions + mock_exists.return_value = False + with pytest.raises(exceptions.InvalidFolderError): + self.node_settings.set_folder('nonexistent-bucket:/', auth=Auth(self.user)) + + @mock.patch('addons.s3.models.bucket_exists') + @mock.patch('addons.s3.models.get_bucket_location_or_error') + def test_set_folder_invalid_auth(self, mock_location, mock_exists): + from addons.base.exceptions import InvalidAuthError + mock_exists.return_value = True + mock_location.side_effect = InvalidAuthError() + with pytest.raises(InvalidAuthError): + self.node_settings.set_folder('valid-bucket:/', auth=Auth(self.user)) + def test_serialize_settings(self): settings = self.node_settings.serialize_waterbutler_settings() expected = { diff --git a/addons/s3/tests/test_view.py b/addons/s3/tests/test_view.py index 56dd8c26f69..dd587cff2d2 100644 --- a/addons/s3/tests/test_view.py +++ b/addons/s3/tests/test_view.py @@ -15,6 +15,7 @@ from addons.s3.utils import validate_bucket_name, validate_bucket_location from website.util import api_url_for from admin.rdm_addons.utils import get_rdm_addon_option +from osf.models import ExternalAccount pytestmark = pytest.mark.django_db @@ -23,8 +24,8 @@ def setUp(self): self.mock_can_list = mock.patch('addons.s3.views.utils.can_list') self.mock_can_list.return_value = True self.mock_can_list.start() - self.mock_uid = mock.patch('addons.s3.views.utils.get_user_info') - self.mock_uid.return_value = {'id': '1234567890', 'display_name': 's3.user'} + self.mock_uid = mock.patch('addons.s3.views.utils.get_user_info', + return_value=mock.Mock(id='1234567890', display_name='s3.user')) self.mock_uid.start() self.mock_exists = mock.patch('addons.s3.views.utils.bucket_exists') self.mock_exists.return_value = True @@ -39,25 +40,25 @@ def tearDown(self): def test_s3_settings_input_empty_keys(self): url = self.project.api_url_for('s3_add_user_account') - rv = self.app.post(url, json={ + rv = self.app.post_json(url, { 'access_key': '', 'secret_key': '' - }, auth=self.user.auth, ) + }, auth=self.user.auth, expect_errors=True) assert rv.status_code == http_status.HTTP_400_BAD_REQUEST assert 'All the fields above are required.' in rv.text def test_s3_settings_input_empty_access_key(self): url = self.project.api_url_for('s3_add_user_account') - rv = self.app.post(url, json={ + rv = self.app.post_json(url, { 'access_key': '', 'secret_key': 'Non-empty-secret-key' - }, auth=self.user.auth, ) + }, auth=self.user.auth, expect_errors=True) assert rv.status_code == http_status.HTTP_400_BAD_REQUEST assert 'All the fields above are required.' in rv.text def test_s3_settings_input_empty_secret_key(self): url = self.project.api_url_for('s3_add_user_account') - rv = self.app.post(url, json={ + rv = self.app.post_json(url, { 'access_key': 'Non-empty-access-key', 'secret_key': '' }, auth=self.user.auth, expect_errors=True) @@ -72,19 +73,77 @@ def test_s3_settings_rdm_addons_denied(self): rdm_addon_option.is_allowed = False rdm_addon_option.save() url = self.project.api_url_for('s3_add_user_account') - rv = self.app.post(url, json={ + rv = self.app.post_json(url, { 'access_key': 'aldkjf', 'secret_key': 'las' }, auth=self.user.auth, expect_errors=True) assert rv.status_code == http_status.HTTP_403_FORBIDDEN assert 'You are prohibited from using this add-on.' in rv.text + # G5: GRDM-53044 provider_id = "{aws_account_id}\t{access_key}" + # setUp already patches get_user_info/can_list as truthy MagicMocks. + # Note: oauth_key/oauth_secret are EncryptedTextField (JWE, non-deterministic), + # so we query by provider_id (plain CharField) and verify via attribute access. + def test_provider_id_format(self): + url = self.project.api_url_for('s3_add_user_account') + access_key = 'AKIAIOSFODNN7EXAMPLE' + rv = self.app.post_json(url, { + 'access_key': access_key, + 'secret_key': 'wJalrXUtnFEMI/K7MDENG', + }, auth=self.user.auth) + assert rv.status_code == http_status.HTTP_200_OK + # provider_id is not encrypted — safe to query + provider_id = f'1234567890\t{access_key}' + account = ExternalAccount.objects.get(provider='s3', provider_id=provider_id) + # provider_id must be "{aws_account_id}\t{access_key}" (GRDM-53044) + assert account.provider_id.endswith(f'\t{access_key}') + assert '\t' in account.provider_id + assert account.oauth_key == access_key + + def test_provider_id_dedup(self): + url = self.project.api_url_for('s3_add_user_account') + access_key = 'AKIAIOSFODNN7EXAMPLE' + provider_id = f'1234567890\t{access_key}' + rv1 = self.app.post_json(url, { + 'access_key': access_key, + 'secret_key': 'secret1', + }, auth=self.user.auth) + assert rv1.status_code == http_status.HTTP_200_OK + # Same AWS account (same provider_id) with new secret → update, no duplicate + rv2 = self.app.post_json(url, { + 'access_key': access_key, + 'secret_key': 'secret2', + }, auth=self.user.auth) + assert rv2.status_code == http_status.HTTP_200_OK + # Query by provider_id (plain CharField) — oauth_key is encrypted (JWE) + accounts = ExternalAccount.objects.filter(provider='s3', provider_id=provider_id) + assert accounts.count() == 1 + assert accounts.first().oauth_secret == 'secret2' + + # G1: missing key field → KeyError → 400 + def test_s3_settings_missing_key_field(self): + url = self.project.api_url_for('s3_add_user_account') + rv = self.app.post_json(url, {}, auth=self.user.auth, expect_errors=True) + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST + + # G2: get_user_info returns None → 400 + "Unable to access account." + @mock.patch('addons.s3.views.utils.get_user_info', return_value=None) + def test_s3_settings_invalid_account(self, mock_get_user_info): + url = self.project.api_url_for('s3_add_user_account') + rv = self.app.post_json(url, { + 'access_key': 'aldkjf', + 'secret_key': 'las', + }, auth=self.user.auth, expect_errors=True) + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST + assert 'Unable to access account.' in rv.text + def test_s3_set_bucket_no_settings(self): user = AuthUserFactory() self.project.add_contributor(user, save=True) url = self.project.api_url_for('s3_set_config') - res = self.app.put( - url, json={'s3_bucket': 'hammertofall'}, auth=user.auth, + res = self.app.put_json( + url, {'s3_bucket': 'hammertofall'}, auth=user.auth, + expect_errors=True, ) assert res.status_code == http_status.HTTP_400_BAD_REQUEST @@ -94,8 +153,9 @@ def test_s3_set_bucket_no_auth(self): user.add_addon('s3') self.project.add_contributor(user, save=True) url = self.project.api_url_for('s3_set_config') - res = self.app.put( - url, json={'s3_bucket': 'hammertofall'}, auth=user.auth, + res = self.app.put_json( + url, {'s3_bucket': 'hammertofall'}, auth=user.auth, + expect_errors=True, ) assert res.status_code == http_status.HTTP_403_FORBIDDEN @@ -105,8 +165,9 @@ def test_s3_set_bucket_registered(self): ) url = registration.api_url_for('s3_set_config') - res = self.app.put( - url, json={'s3_bucket': 'hammertofall'}, auth=self.user.auth, + res = self.app.put_json( + url, {'s3_bucket': 'hammertofall'}, auth=self.user.auth, + expect_errors=True, ) assert res.status_code == http_status.HTTP_400_BAD_REQUEST @@ -114,10 +175,10 @@ def test_s3_set_bucket_registered(self): @mock.patch('addons.s3.views.utils.can_list', return_value=False) def test_user_settings_cant_list(self, mock_can_list): url = api_url_for('s3_add_user_account') - rv = self.app.post(url, json={ + rv = self.app.post_json(url, { 'access_key': 'aldkjf', 'secret_key': 'las' - }, auth=self.user.auth) + }, auth=self.user.auth, expect_errors=True) assert 'Unable to list buckets.' in rv.text assert rv.status_code == http_status.HTTP_400_BAD_REQUEST @@ -130,7 +191,7 @@ def test_s3_remove_node_settings_owner(self): def test_s3_remove_node_settings_unauthorized(self): url = self.node_settings.owner.api_url_for('s3_deauthorize_node') - ret = self.app.delete(url, auth=None, ) + ret = self.app.delete(url, auth=None, expect_errors=True) assert ret.status_code == 401 @@ -149,7 +210,7 @@ def test_s3_get_node_settings_owner(self): def test_s3_get_node_settings_unauthorized(self): url = self.node_settings.owner.api_url_for('s3_get_config') unauthorized = AuthUserFactory() - ret = self.app.get(url, auth=unauthorized.auth, ) + ret = self.app.get(url, auth=unauthorized.auth, expect_errors=True) assert ret.status_code == 403 @@ -167,7 +228,7 @@ def test_set_config(self, mock_location, mock_exists): mock_location.return_value = '' self.node_settings.set_auth(self.external_account, self.user) url = self.project.api_url_for(f'{self.ADDON_SHORT_NAME}_set_config') - res = self.app.put(url, json={ + res = self.app.put_json(url, { 'selected': self.folder }, auth=self.user.auth) assert res.status_code == http_status.HTTP_200_OK @@ -265,9 +326,9 @@ def test_create_bucket_pass(self, mock_names, mock_make): 'doesntevenmatter' ] url = self.project.api_url_for('create_bucket') - ret = self.app.post( + ret = self.app.post_json( url, - json={ + { 'bucket_name': 'doesntevenmatter', 'bucket_location': '', }, @@ -283,19 +344,69 @@ def test_create_bucket_fail(self, mock_make): mock_make.side_effect = error url = f'/api/v1/project/{self.project._id}/s3/newbucket/' - ret = self.app.post(url, json={'bucket_name': 'doesntevenmatter'}, auth=self.user.auth) + ret = self.app.post_json(url, {'bucket_name': 'doesntevenmatter'}, auth=self.user.auth, + expect_errors=True) assert ret.text == '{"message": "Unable to locate credentials", "title": "Problem connecting to S3"}' @mock.patch('addons.s3.views.utils.create_bucket') def test_bad_location_fails(self, mock_make): url = f'/api/v1/project/{self.project._id}/s3/newbucket/' - ret = self.app.post( + ret = self.app.post_json( url, - json={ + { 'bucket_name': 'doesntevenmatter', 'bucket_location': 'not a real bucket location', }, auth=self.user.auth, + expect_errors=True, ) assert ret.text == '{"message": "That bucket location is not valid.", "title": "Invalid bucket location"}' + + # G4: invalid bucket name → 400 (name check fires before any S3 call) + def test_create_bucket_bad_name(self): + url = f'/api/v1/project/{self.project._id}/s3/newbucket/' + ret = self.app.post_json( + url, + { + 'bucket_name': 'no', + 'bucket_location': '', + }, + auth=self.user.auth, + expect_errors=True, + ) + assert ret.status_code == http_status.HTTP_400_BAD_REQUEST + assert 'That bucket name is not valid.' in ret.text + + @mock.patch('addons.s3.views.utils.create_bucket') + def test_create_bucket_client_error(self, mock_make): + from botocore.exceptions import ClientError + error = ClientError( + {'Error': {'Code': 'BucketAlreadyExists', 'Message': 'The requested bucket name is not available.'}}, + 'CreateBucket' + ) + mock_make.side_effect = error + url = f'/api/v1/project/{self.project._id}/s3/newbucket/' + ret = self.app.post_json( + url, + {'bucket_name': 'valid-bucket-name', 'bucket_location': ''}, + auth=self.user.auth, + expect_errors=True, + ) + assert ret.status_code == http_status.HTTP_400_BAD_REQUEST + assert ret.json['title'] == "Problem creating bucket 'valid-bucket-name'" + assert 'BucketAlreadyExists' in ret.json['message'] + + +class TestS3Utils: + """Pure unit tests for utility functions — no DB required.""" + + # G7: bucket_exists with empty name → False (early exit, no S3 call) + def test_bucket_exists_empty_name(self): + from addons.s3.utils import bucket_exists + assert bucket_exists('key', 'secret', '') is False + + # G8: can_list(None, None) → False (early exit, no S3 call) + def test_can_list_no_keys(self): + from addons.s3.utils import can_list + assert can_list(None, None) is False From b6843500490f301a847d72aa5e4ff9f26a54d22f Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Fri, 2 Oct 2026 15:29:42 +0900 Subject: [PATCH 26/41] fix(s3): preserve bucket names as given and tolerate a missing owner display name - stop lower-casing the bucket name in the settings check. S3 bucket names are case-sensitive in the legacy us-east-1 naming rules, so folding the case can point the addon at a different bucket; validate_bucket_name already rejects the names that are actually invalid - fall back gracefully when the owner returned by GetUserInfo has no display name, instead of raising while building the account label - keep the Owner entry of the admin storage-location response in its original shape so that the existing consumers keep working --- addons/s3/tests/test_model.py | 16 ++++++++++++++++ addons/s3/tests/test_view.py | 6 ++++++ addons/s3/utils.py | 8 ++------ admin/rdm_custom_storage_location/utils.py | 5 ++++- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/addons/s3/tests/test_model.py b/addons/s3/tests/test_model.py index ac148c59812..9e22b3f591f 100644 --- a/addons/s3/tests/test_model.py +++ b/addons/s3/tests/test_model.py @@ -122,3 +122,19 @@ def test_serialize_settings(self): 'id': 'bucket_name:/path_goes_here/with_folder_id' } assert settings == expected + + @pytest.mark.parametrize('folder_id, expected_bucket', [ + ('bucket', 'bucket'), + ('bucket:/', 'bucket'), + ('bucket:/prefix/', 'bucket'), + ]) + def test_serialize_settings_supports_legacy_and_prefixed_folder_ids(self, folder_id, expected_bucket): + self.node_settings.folder_id = folder_id + + settings = self.node_settings.serialize_waterbutler_settings() + + assert settings == { + 'bucket': expected_bucket, + 'encrypt_uploads': self.node_settings.encrypt_uploads, + 'id': folder_id, + } diff --git a/addons/s3/tests/test_view.py b/addons/s3/tests/test_view.py index dd587cff2d2..8cc8a6a5452 100644 --- a/addons/s3/tests/test_view.py +++ b/addons/s3/tests/test_view.py @@ -401,6 +401,12 @@ def test_create_bucket_client_error(self, mock_make): class TestS3Utils: """Pure unit tests for utility functions — no DB required.""" + def test_owner_from_dict_without_display_name(self): + from addons.s3.utils import Owner + owner = Owner.from_dict({'ID': 'owner-id'}) + assert owner.id == 'owner-id' + assert owner.display_name == 'owner-id' + # G7: bucket_exists with empty name → False (early exit, no S3 call) def test_bucket_exists_empty_name(self): from addons.s3.utils import bucket_exists diff --git a/addons/s3/utils.py b/addons/s3/utils.py index 73d53ebf18e..a3ed3ab8ad5 100644 --- a/addons/s3/utils.py +++ b/addons/s3/utils.py @@ -84,8 +84,6 @@ def bucket_exists(access_key, secret_key, bucket_name): """ if not bucket_name: return False - # bucket names are dns-compliant, therefore must be lowercase - bucket_name = bucket_name.lower() try: # Will raise an exception if bucket_name doesn't exist @@ -121,7 +119,8 @@ class Owner: @classmethod def from_dict(cls, data): - return cls(data['DisplayName'], data['ID']) + owner_id = data['ID'] + return cls(data.get('DisplayName') or owner_id, owner_id) def get_user_info(access_key: str, secret_key: str) -> Optional[Owner]: @@ -139,9 +138,6 @@ def get_user_info(access_key: str, secret_key: str) -> Optional[Owner]: def get_bucket_location_or_error(access_key, secret_key, bucket_name): """Returns the location of a bucket or raises AddonError """ - # bucket names are dns-compliant, therefore must be lowercase - bucket_name = bucket_name.lower() - try: # Will raise an exception if bucket_name doesn't exist return connect_s3(access_key, secret_key).get_bucket_location(Bucket=bucket_name)['LocationConstraint'] diff --git a/admin/rdm_custom_storage_location/utils.py b/admin/rdm_custom_storage_location/utils.py index df350ed2ae1..ae6ab5209d6 100644 --- a/admin/rdm_custom_storage_location/utils.py +++ b/admin/rdm_custom_storage_location/utils.py @@ -261,7 +261,10 @@ def test_s3_connection(access_key, secret_key, bucket): s3_response = { 'id': user_info.id, 'display_name': user_info.display_name, - 'Owner': user_info.Owner, + 'Owner': { + 'ID': user_info.id, + 'DisplayName': user_info.display_name, + }, } return ({ From abb5a78de229f86f0daeb4e506590cf72ed4ef1c Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Fri, 2 Oct 2026 15:29:53 +0900 Subject: [PATCH 27/41] test(s3): run the folder_id cases in a loop instead of pytest.mark.parametrize pytest.mark.parametrize has no effect on a method of a unittest.TestCase: the case is collected once and the extra arguments are never supplied, so test_serialize_settings_supports_legacy_and_prefixed_folder_ids errored on a missing fixture instead of checking the three folder_id forms. Iterate over the cases inside the test body instead. --- addons/s3/tests/test_model.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/addons/s3/tests/test_model.py b/addons/s3/tests/test_model.py index 9e22b3f591f..8e549c5d1ae 100644 --- a/addons/s3/tests/test_model.py +++ b/addons/s3/tests/test_model.py @@ -123,18 +123,20 @@ def test_serialize_settings(self): } assert settings == expected - @pytest.mark.parametrize('folder_id, expected_bucket', [ - ('bucket', 'bucket'), - ('bucket:/', 'bucket'), - ('bucket:/prefix/', 'bucket'), - ]) - def test_serialize_settings_supports_legacy_and_prefixed_folder_ids(self, folder_id, expected_bucket): - self.node_settings.folder_id = folder_id - - settings = self.node_settings.serialize_waterbutler_settings() - - assert settings == { - 'bucket': expected_bucket, - 'encrypt_uploads': self.node_settings.encrypt_uploads, - 'id': folder_id, - } + def test_serialize_settings_supports_legacy_and_prefixed_folder_ids(self): + cases = [ + ('bucket', 'bucket'), + ('bucket:/', 'bucket'), + ('bucket:/prefix/', 'bucket'), + ] + + for folder_id, expected_bucket in cases: + self.node_settings.folder_id = folder_id + + settings = self.node_settings.serialize_waterbutler_settings() + + assert settings == { + 'bucket': expected_bucket, + 'encrypt_uploads': self.node_settings.encrypt_uploads, + 'id': folder_id, + } From 14a80274321b12263bc1f94d9528f8433526d712 Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Thu, 1 Oct 2026 09:43:06 +0900 Subject: [PATCH 28/41] Report us-east-1 as the empty-string bucket location get_bucket_location() returns a null LocationConstraint for us-east-1, but BUCKET_LOCATIONS keys that region as '', so the null fell through to set_folder and labelled the bucket '(None)' in the node settings UI. --- addons/s3/tests/test_view.py | 17 +++++++++++++++++ addons/s3/utils.py | 5 +++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/addons/s3/tests/test_view.py b/addons/s3/tests/test_view.py index 8cc8a6a5452..61ed13c669b 100644 --- a/addons/s3/tests/test_view.py +++ b/addons/s3/tests/test_view.py @@ -416,3 +416,20 @@ def test_bucket_exists_empty_name(self): def test_can_list_no_keys(self): from addons.s3.utils import can_list assert can_list(None, None) is False + + # G9: us-east-1 is reported as a null LocationConstraint, but BUCKET_LOCATIONS + # keys that region as ''. Passing the null through labels the bucket '(None)'. + @mock.patch('addons.s3.utils.connect_s3') + def test_null_location_constraint_becomes_empty_string(self, mock_connect): + from addons.s3.utils import get_bucket_location_or_error + mock_connect.return_value.get_bucket_location.return_value = {'LocationConstraint': None} + assert get_bucket_location_or_error('key', 'secret', 'bucket') == '' + + # G10: every other region reports its own name and must survive untouched + @mock.patch('addons.s3.utils.connect_s3') + def test_named_region_is_passed_through(self, mock_connect): + from addons.s3.utils import get_bucket_location_or_error + mock_connect.return_value.get_bucket_location.return_value = { + 'LocationConstraint': 'ap-northeast-1' + } + assert get_bucket_location_or_error('key', 'secret', 'bucket') == 'ap-northeast-1' diff --git a/addons/s3/utils.py b/addons/s3/utils.py index a3ed3ab8ad5..8992ad9dd27 100644 --- a/addons/s3/utils.py +++ b/addons/s3/utils.py @@ -139,8 +139,9 @@ def get_bucket_location_or_error(access_key, secret_key, bucket_name): """Returns the location of a bucket or raises AddonError """ try: - # Will raise an exception if bucket_name doesn't exist - return connect_s3(access_key, secret_key).get_bucket_location(Bucket=bucket_name)['LocationConstraint'] + # Will raise an exception if bucket_name doesn't exist. + # us-east-1 is reported as a null LocationConstraint; BUCKET_LOCATIONS keys it as ''. + return connect_s3(access_key, secret_key).get_bucket_location(Bucket=bucket_name)['LocationConstraint'] or '' except exceptions.NoCredentialsError: raise InvalidAuthError() except exceptions.ClientError: From 10873ee6c07272a391e6dfb6a97ccac285374778 Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Thu, 1 Oct 2026 09:43:27 +0900 Subject: [PATCH 29/41] Translate list_objects failures into HTTPError in get_bucket_prefixes The ListObjects call was unguarded, so expired or revoked keys and a missing bucket escaped as a raw botocore exception and reached the user as a 500 with no actionable message. Map NoCredentialsError to 403 and reuse get_status_for_error so a ClientError keeps the status S3 returned. --- addons/s3/tests/test_view.py | 57 ++++++++++++++++++++++++++++++++++++ addons/s3/utils.py | 8 ++++- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/addons/s3/tests/test_view.py b/addons/s3/tests/test_view.py index 61ed13c669b..46e0aced70f 100644 --- a/addons/s3/tests/test_view.py +++ b/addons/s3/tests/test_view.py @@ -433,3 +433,60 @@ def test_named_region_is_passed_through(self, mock_connect): 'LocationConstraint': 'ap-northeast-1' } assert get_bucket_location_or_error('key', 'secret', 'bucket') == 'ap-northeast-1' + + # G11: the queried prefix comes back in CommonPrefixes and must not list itself + @mock.patch('addons.s3.utils.boto3.client') + def test_get_bucket_prefixes_skips_the_queried_prefix(self, mock_client): + from addons.s3.utils import get_bucket_prefixes + mock_client.return_value.list_objects.return_value = { + 'CommonPrefixes': [ + {'Prefix': 'parent/'}, + {'Prefix': 'parent/child/'}, + ] + } + + folders = get_bucket_prefixes('key', 'secret', 'parent/', 'bucket') + + assert len(folders) == 1 + assert folders[0]['name'] == 'child' + assert folders[0]['id'] == 'bucket:/parent/child/' + assert folders[0]['path'] == 'parent/child/' + assert folders[0]['kind'] == 'folder' + + # G12: a bucket with no sub-prefixes omits CommonPrefixes entirely + @mock.patch('addons.s3.utils.boto3.client') + def test_get_bucket_prefixes_no_common_prefixes(self, mock_client): + from addons.s3.utils import get_bucket_prefixes + mock_client.return_value.list_objects.return_value = {} + assert get_bucket_prefixes('key', 'secret', '', 'bucket') == [] + + # G13: boto failures must become HTTPError, not an uncaught 500 + @mock.patch('addons.s3.utils.boto3.client') + def test_get_bucket_prefixes_missing_credentials_raises_403(self, mock_client): + from framework.exceptions import HTTPError + from addons.s3.utils import get_bucket_prefixes + mock_client.return_value.list_objects.side_effect = NoCredentialsError() + + with pytest.raises(HTTPError) as exc_info: + get_bucket_prefixes('key', 'secret', '', 'bucket') + + assert exc_info.value.code == http_status.HTTP_403_FORBIDDEN + + # G14: a ClientError keeps whatever status S3 itself returned + @mock.patch('addons.s3.utils.boto3.client') + def test_get_bucket_prefixes_client_error_keeps_status(self, mock_client): + from botocore.exceptions import ClientError + from framework.exceptions import HTTPError + from addons.s3.utils import get_bucket_prefixes + mock_client.return_value.list_objects.side_effect = ClientError( + { + 'Error': {'Code': 'NoSuchBucket', 'Message': 'denied'}, + 'ResponseMetadata': {'HTTPStatusCode': http_status.HTTP_404_NOT_FOUND}, + }, + 'ListObjects', + ) + + with pytest.raises(HTTPError) as exc_info: + get_bucket_prefixes('key', 'secret', '', 'bucket') + + assert exc_info.value.code == http_status.HTTP_404_NOT_FOUND diff --git a/addons/s3/utils.py b/addons/s3/utils.py index 8992ad9dd27..f0cb70da46c 100644 --- a/addons/s3/utils.py +++ b/addons/s3/utils.py @@ -155,7 +155,13 @@ def get_bucket_prefixes(access_key, secret_key, prefix, bucket_name): aws_secret_access_key=secret_key ) - result = s3.list_objects(Bucket=bucket_name, Prefix=prefix, Delimiter='/') + try: + result = s3.list_objects(Bucket=bucket_name, Prefix=prefix, Delimiter='/') + except exceptions.NoCredentialsError: + raise HTTPError(http_status.HTTP_403_FORBIDDEN) + except exceptions.ClientError as e: + raise HTTPError(get_status_for_error(e)) + folders = [] for common_prefixes in result.get('CommonPrefixes', []): key_name = common_prefixes.get('Prefix') From 94d57ea2464e912e4be65d2705f7c22ba8543138 Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Thu, 1 Oct 2026 09:43:44 +0900 Subject: [PATCH 30/41] Catch BotoCoreError when creating a bucket create_bucket only handled NoCredentialsError and ClientError, so the non-response failures - bad parameters, an unreachable endpoint - left the user with a 500. ClientError is not a BotoCoreError subclass and NoCredentialsError is matched earlier, so the existing clauses keep their behaviour. --- addons/s3/tests/test_view.py | 17 +++++++++++++++++ addons/s3/views.py | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/addons/s3/tests/test_view.py b/addons/s3/tests/test_view.py index 46e0aced70f..c60d796b91d 100644 --- a/addons/s3/tests/test_view.py +++ b/addons/s3/tests/test_view.py @@ -397,6 +397,23 @@ def test_create_bucket_client_error(self, mock_make): assert ret.json['title'] == "Problem creating bucket 'valid-bucket-name'" assert 'BucketAlreadyExists' in ret.json['message'] + @mock.patch('addons.s3.views.utils.create_bucket') + def test_create_bucket_boto_core_error(self, mock_make): + # BotoCoreError covers the non-response failures (bad parameters, endpoint + # unreachable, ...). Without the catchall these surface as a 500. + from botocore.exceptions import ParamValidationError + mock_make.side_effect = ParamValidationError(report='Invalid bucket configuration') + url = f'/api/v1/project/{self.project._id}/s3/newbucket/' + ret = self.app.post_json( + url, + {'bucket_name': 'valid-bucket-name', 'bucket_location': ''}, + auth=self.user.auth, + expect_errors=True, + ) + assert ret.status_code == http_status.HTTP_400_BAD_REQUEST + assert ret.json['title'] == 'Error connecting to S3' + assert 'Invalid bucket configuration' in ret.json['message'] + class TestS3Utils: """Pure unit tests for utility functions — no DB required.""" diff --git a/addons/s3/views.py b/addons/s3/views.py index 1627bcf1a7f..5bccd9b3d3a 100644 --- a/addons/s3/views.py +++ b/addons/s3/views.py @@ -162,5 +162,12 @@ def create_bucket(auth, node_addon, **kwargs): 'message': str(e), 'title': f"Problem creating bucket '{bucket_name}'", }, http_status.HTTP_400_BAD_REQUEST + except exceptions.BotoCoreError as e: + # Base class catchall. ClientError is not a BotoCoreError, and NoCredentialsError + # is handled above, so neither of the clauses above is shadowed by this one. + return { + 'message': str(e), + 'title': 'Error connecting to S3', + }, http_status.HTTP_400_BAD_REQUEST return {} From 34e0cb33ac4ef40d4244befa12953be61a281caf Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:58:18 +0900 Subject: [PATCH 31/41] Make the JAIRO Cloud index picker searchable by full index path because repositories with many indices were hard to scroll through --- addons/weko/static/wekoNodeConfig.js | 40 +++++++++++-------- addons/weko/templates/weko_node_settings.mako | 5 ++- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/addons/weko/static/wekoNodeConfig.js b/addons/weko/static/wekoNodeConfig.js index 7b1fcd0a430..5e34a8314b0 100644 --- a/addons/weko/static/wekoNodeConfig.js +++ b/addons/weko/static/wekoNodeConfig.js @@ -8,6 +8,7 @@ var bootbox = require('bootbox'); var Raven = require('raven-js'); var $osf = require('js/osfHelpers'); +require('js/rdmSelect2'); var $modal = $('#wekoInputCredentials'); @@ -29,31 +30,38 @@ function _getIndexById(indices, id) { return null; } -function _getIndexDisplayTitle(index, level) { - if (level === 0) { - return index.title; - } - var prefix = ''; - for (var i = 0; i < level; i ++) { - prefix += ' '; - } - return prefix + '- ' + index.title; -} - -function _flattenIndices(indices, level) { +function _flattenIndices(indices, ancestorTitles) { const r = []; indices.forEach(function(data) { - const displayTitle = _getIndexDisplayTitle(data, level); + const titles = ancestorTitles.concat([data.title]); r.push(Object.assign({ - displayTitle: displayTitle, + displayTitle: titles.join(' / '), }, data)); - _flattenIndices(data.children, level + 1).forEach(function(child) { + _flattenIndices(data.children, titles).forEach(function(child) { r.push(child); }); }); return r; } +ko.bindingHandlers.wekoIndexSelect2 = { + // select2 mirrors the keeps form-control because existing selectors rely on it; + // copied onto the select2 box it would draw a second border + adaptContainerCssClass: function() { + return null; + } + }); + }, + update: function(element, valueAccessor) { + $(element).select2('val', ko.unwrap(valueAccessor())); + } +}; + function ViewModel(url) { var self = this; @@ -172,7 +180,7 @@ function ViewModel(url) { }); self.flattenIndices = ko.pureComputed(function() { - return _flattenIndices(self.indices(), 0); + return _flattenIndices(self.indices(), []); }); // Flashed messages diff --git a/addons/weko/templates/weko_node_settings.mako b/addons/weko/templates/weko_node_settings.mako index 1f40c270815..0d92208468c 100644 --- a/addons/weko/templates/weko_node_settings.mako +++ b/addons/weko/templates/weko_node_settings.mako @@ -59,13 +59,14 @@
      -
      +
      ${_("Index:")}
      From 6bf603431e63d57fa898dd1e2c6bd5b0823f2728 Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:58:24 +0900 Subject: [PATCH 32/41] Show the JAIRO Cloud index save confirmation, which threw on folder helpers this view model never had --- addons/weko/static/wekoNodeConfig.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/weko/static/wekoNodeConfig.js b/addons/weko/static/wekoNodeConfig.js index 5e34a8314b0..f0f3fc4ab98 100644 --- a/addons/weko/static/wekoNodeConfig.js +++ b/addons/weko/static/wekoNodeConfig.js @@ -126,7 +126,7 @@ function ViewModel(url) { setInfoSuccess: ko.pureComputed(function() { var filesUrl = window.contextVars.node.urls.web + 'files/'; return sprintf(_('Successfully linked index "%1$s". Go to the Files page to view your content.'), - $osf.htmlEscape(self.options.decodeFolder(self.folder().name)), filesUrl); + $osf.htmlEscape(self.savedIndexTitle()), filesUrl); }), setIndexError: ko.pureComputed(function() { return sprintf(_('Could not connect to this index. Please refresh the page or ' + From dc29e79acc0ebb0c9321b30e587e34b273fb88e8 Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:58:33 +0900 Subject: [PATCH 33/41] Remove the unused connectAccount from the JAIRO Cloud node config, which referenced an undefined message --- addons/weko/static/wekoNodeConfig.js | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/addons/weko/static/wekoNodeConfig.js b/addons/weko/static/wekoNodeConfig.js index f0f3fc4ab98..ce0df78fd8f 100644 --- a/addons/weko/static/wekoNodeConfig.js +++ b/addons/weko/static/wekoNodeConfig.js @@ -345,20 +345,6 @@ ViewModel.prototype.onImportError = function(xhr, status, error) { }); }; -/** - * Allows a user to create an access token from the nodeSettings page - */ -ViewModel.prototype.connectAccount = function() { - var self = this; - - window.oauthComplete = function(res) { - // Update view model based on response - self.changeMessage(self.messages.connectAccountSuccess(), 'text-success', 3000); - self.importAuth.call(self); - }; - window.open(self.urls().auth); -}; - ViewModel.prototype.connectExistingAccount = function(account_id) { var self = this; From 787e30e965c174cd17991564fb8170353b95fc06 Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:38:45 +0900 Subject: [PATCH 34/41] Expand the children WEKO hides behind an index's more entry so every index can be found and deposited to --- addons/weko/client.py | 27 ++++++++++++++++- addons/weko/tests/test_client.py | 50 ++++++++++++++++++++++++++++++++ addons/weko/tests/utils.py | 3 -- 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/addons/weko/client.py b/addons/weko/client.py index f7b00fcf459..84bd0c374af 100644 --- a/addons/weko/client.py +++ b/addons/weko/client.py @@ -15,6 +15,19 @@ def _flatten_indices(indices): return r +def _get_truncated_index_ids(descs): + # WEKO replaces the children beyond an index's display number with a "more" entry + # unless the index is listed in more_ids + r = [] + for desc in descs: + if 'children' not in desc: + continue + if any(child['id'] == 'more' for child in desc['children']): + r.append(str(desc['id'])) + r += _get_truncated_index_ids(desc['children']) + return r + + def _is_valid_index(desc): if 'name' in desc and 'id' in desc: return True @@ -62,7 +75,19 @@ def get_indices(self): """ Get all indices from the WEKO. """ - root = self._get('api/tree?action=browsing') + more_ids = [] + while True: + path = 'api/tree?action=browsing' + if more_ids: + path += '&more_ids=' + '/'.join(more_ids) + root = self._get(path) + truncated_ids = _get_truncated_index_ids(root) + if not truncated_ids: + break + unexpanded_ids = set(truncated_ids) & set(more_ids) + if unexpanded_ids: + raise ValueError(f'WEKO did not expand indices: {unexpanded_ids}') + more_ids += truncated_ids indices = [] for desc in root: if not _is_valid_index(desc): diff --git a/addons/weko/tests/test_client.py b/addons/weko/tests/test_client.py index d616196ec50..f1e600e644b 100644 --- a/addons/weko/tests/test_client.py +++ b/addons/weko/tests/test_client.py @@ -19,6 +19,36 @@ def mock_requests_get(url, **kwargs): return utils.mock_response_404 +def _fake_truncated_indices(expanded_ids): + grandchildren = [{'id': '300', 'name': 'Grandchild 1', 'children': []}] + if '201' in expanded_ids: + grandchildren.append({'id': '301', 'name': 'Grandchild 2', 'children': []}) + else: + grandchildren.append({'id': 'more'}) + children = [{'id': '200', 'name': 'Child 1', 'children': []}] + if '100' in expanded_ids: + children.append({'id': '201', 'name': 'Child 2', 'children': grandchildren}) + else: + children.append({'id': 'more'}) + return [{'id': '100', 'name': 'Sample Index', 'children': children}] + + +def mock_requests_get_truncated(url, **kwargs): + prefix = 'https://test.sample.nii.ac.jp/api/tree?action=browsing' + if url == prefix: + return utils.MockResponse(_fake_truncated_indices([]), 200) + if url.startswith(prefix + '&more_ids='): + expanded_ids = url[len(prefix + '&more_ids='):].split('/') + return utils.MockResponse(_fake_truncated_indices(expanded_ids), 200) + return utils.mock_response_404 + + +def mock_requests_get_never_expanded(url, **kwargs): + if url.startswith('https://test.sample.nii.ac.jp/api/tree?action=browsing'): + return utils.MockResponse(_fake_truncated_indices([]), 200) + return utils.mock_response_404 + + class TestWEKOClient(OsfTestCase): def setUp(self): self.host = utils.fake_weko_host @@ -35,6 +65,26 @@ def test_weko_get_indices(self, get_req_mock): assert_equal(indices[0].title, 'Sample Index') assert_equal(indices[0].identifier, 100) + @mock.patch('requests.get', side_effect=mock_requests_get_truncated) + def test_weko_get_indices_expands_truncated_children(self, get_req_mock): + indices = self.conn.get_indices() + + titles = [i.title for i in client._flatten_indices(indices)] + assert_equal(titles, ['Sample Index', 'Child 1', 'Child 2', 'Grandchild 1', 'Grandchild 2']) + assert_equal( + [c[0][0] for c in get_req_mock.call_args_list], + [ + 'https://test.sample.nii.ac.jp/api/tree?action=browsing', + 'https://test.sample.nii.ac.jp/api/tree?action=browsing&more_ids=100', + 'https://test.sample.nii.ac.jp/api/tree?action=browsing&more_ids=100/201', + ] + ) + + @mock.patch('requests.get', side_effect=mock_requests_get_never_expanded) + def test_weko_get_indices_fails_when_weko_keeps_children_truncated(self, get_req_mock): + with assert_raises(ValueError): + self.conn.get_indices() + @mock.patch('requests.get', side_effect=mock_requests_get) def test_weko_get_index_by_id(self, get_req_mock): index = self.conn.get_index_by_id(100) diff --git a/addons/weko/tests/utils.py b/addons/weko/tests/utils.py index 076064d897a..f04a75ae656 100644 --- a/addons/weko/tests/utils.py +++ b/addons/weko/tests/utils.py @@ -11,9 +11,6 @@ 'id': 100, 'name': 'Sample Index', 'children': [ - { - 'id': 'more', - }, { 'id': 'dummy', } From 5ff320b825639acc41d47cf028cb8c65789569bf Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:34:06 +0900 Subject: [PATCH 35/41] Show each JAIRO Cloud index's ID in the picker and match it in search because index names are not unique --- addons/weko/static/wekoNodeConfig.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/addons/weko/static/wekoNodeConfig.js b/addons/weko/static/wekoNodeConfig.js index ce0df78fd8f..36965c5b6c0 100644 --- a/addons/weko/static/wekoNodeConfig.js +++ b/addons/weko/static/wekoNodeConfig.js @@ -44,16 +44,33 @@ function _flattenIndices(indices, ancestorTitles) { return r; } +function _withIndexId(title, id) { + return title + ' (ID:' + id + ')'; +} + ko.bindingHandlers.wekoIndexSelect2 = { // select2 mirrors the relies on it $(element).select2({ width: '100%', // The , so the element must be up to date first after: ['options', 'value'], - init: function(element) { + init: function(element, valueAccessor) { const defaults = $.fn.select2.defaults; - // The ID is shown and matched here; the option text stays the plain path - // because label-based selection of the relies on it $(element).select2({ width: '100%', // The
      diff --git a/addons/weko/tests/test_client.py b/addons/weko/tests/test_client.py index f1e600e644b..a624e293adc 100644 --- a/addons/weko/tests/test_client.py +++ b/addons/weko/tests/test_client.py @@ -85,6 +85,17 @@ def test_weko_get_indices_fails_when_weko_keeps_children_truncated(self, get_req with assert_raises(ValueError): self.conn.get_indices() + @mock.patch('requests.get', side_effect=mock_requests_get) + def test_weko_get_indices_asks_for_the_given_language(self, get_req_mock): + self.conn.get_indices(accept_language='ja') + + get_req_mock.assert_called_once_with( + 'https://test.sample.nii.ac.jp/api/tree?action=browsing', + auth=(None, None), + headers={'Accept-Language': 'ja'}, + timeout=client.DEFAULT_TIMEOUT, + ) + @mock.patch('requests.get', side_effect=mock_requests_get) def test_weko_get_index_by_id(self, get_req_mock): index = self.conn.get_index_by_id(100) diff --git a/addons/weko/tests/test_serializer.py b/addons/weko/tests/test_serializer.py index 51d56e40930..6fbf5e5a1fe 100644 --- a/addons/weko/tests/test_serializer.py +++ b/addons/weko/tests/test_serializer.py @@ -82,6 +82,42 @@ def test_serialize_settings_authorized(self): assert_equal(serialized['ownerName'], self.user_settings.owner.fullname) assert_in('savedIndex', serialized) + def test_serialize_settings_names_indices_in_the_page_language(self): + def requests_get(url, **kwargs): + if kwargs.get('headers', {}).get('Accept-Language') == 'ja': + return utils.MockResponse([{'id': 100, 'name': 'サンプルインデックス'}], 200) + return utils.MockResponse([{'id': 100, 'name': 'Sample Index'}], 200) + + with mock.patch.object(type(self.node_settings), 'has_auth', return_value=True), \ + mock.patch.object(WEKOSerializer, 'credentials_are_valid', return_value=True), \ + mock.patch('requests.get', side_effect=requests_get), \ + self.app.app.test_request_context(headers={'Accept-Language': 'ja-JP'}): + serialized = self.ser.serialize_settings(self.node_settings, self.user, self.client) + + assert_equal(serialized['indices'], [{ + 'title': 'Sample Index', + 'localizedTitle': 'サンプルインデックス', + 'id': 100, + 'children': [], + }]) + + def test_serialize_settings_falls_back_to_the_default_name_and_warns(self): + def requests_get(url, **kwargs): + if kwargs.get('headers', {}).get('Accept-Language') == 'ja': + return utils.MockResponse([{'id': 100, 'name': 'サンプルインデックス'}], 200) + return utils.MockResponse([{'id': 100, 'name': 'Sample Index'}, {'id': 101, 'name': 'New Index'}], 200) + + with mock.patch.object(type(self.node_settings), 'has_auth', return_value=True), \ + mock.patch.object(WEKOSerializer, 'credentials_are_valid', return_value=True), \ + mock.patch('requests.get', side_effect=requests_get), \ + self.app.app.test_request_context(headers={'Accept-Language': 'ja'}), \ + self.assertLogs('addons.weko.serializer', level='WARNING') as logs: + serialized = self.ser.serialize_settings(self.node_settings, self.user, self.client) + + assert_equal([i['localizedTitle'] for i in serialized['indices']], ['サンプルインデックス', 'New Index']) + assert_equal(len(logs.records), 1) + assert_in('101', logs.records[0].getMessage()) + def test_serialize_settings_authorized_folder_is_set(self): pass From 0ab1625998c8e16504ac6bf02f8fec69096e64e7 Mon Sep 17 00:00:00 2001 From: chiku Date: Tue, 13 Oct 2026 15:13:59 +0900 Subject: [PATCH 37/41] Add MATLAB release R2026b Register R2026b at the head of MATLAB_RELEASES and in MATLAB_PRODUCTNAMES_MAP, and add its mpm input file. It makes the release available in the BinderHub addon. Co-Authored-By: Claude Fable 5.1 --- addons/binderhub/settings/defaults.py | 2 + .../settings/mpm_inputs/mpm_input_r2026b.txt | 514 ++++++++++++++++++ 2 files changed, 516 insertions(+) create mode 100644 addons/binderhub/settings/mpm_inputs/mpm_input_r2026b.txt diff --git a/addons/binderhub/settings/defaults.py b/addons/binderhub/settings/defaults.py index 0a59f0fe19c..b22b155c219 100644 --- a/addons/binderhub/settings/defaults.py +++ b/addons/binderhub/settings/defaults.py @@ -44,6 +44,7 @@ ] MATLAB_RELEASES = [ + 'R2026b', 'R2026a', 'R2025b', 'R2025a', @@ -56,6 +57,7 @@ ] MATLAB_PRODUCTNAMES_MAP = { + 'R2026b': gen_product_name_list('addons/binderhub/settings/mpm_inputs/mpm_input_r2026b.txt'), 'R2026a': gen_product_name_list('addons/binderhub/settings/mpm_inputs/mpm_input_r2026a.txt'), 'R2025b': gen_product_name_list('addons/binderhub/settings/mpm_inputs/mpm_input_r2025b.txt'), 'R2025a': gen_product_name_list('addons/binderhub/settings/mpm_inputs/mpm_input_r2025a.txt'), diff --git a/addons/binderhub/settings/mpm_inputs/mpm_input_r2026b.txt b/addons/binderhub/settings/mpm_inputs/mpm_input_r2026b.txt new file mode 100644 index 00000000000..f6fc9260862 --- /dev/null +++ b/addons/binderhub/settings/mpm_inputs/mpm_input_r2026b.txt @@ -0,0 +1,514 @@ +######################################################################## +## Configuration File for Installing or Downloading R2026b MathWorks Products +######################################################################## +## +## Use this file to configure the installation or download of MathWorks +## products and support packages from the command line using the +## MATLAB Package Manager (mpm). +## +## To configure your MATLAB installation or download: +## +## 1. Set configuration parameters by uncommenting lines that +## start with a single '#' and updating the values. The +## comments above each parameter describe the valid values. +## +## 2. Run mpm from the command line, using the --inputfile option +## to specify the full path to this configuration file. +## +## Install products: +## mpm install --inputfile +## +## Download products without installing: +## mpm download --inputfile +## +## You can download template input files for all supported releases from +## https://www.mathworks.com/mpm_input_files +## +######################################################################## + + +######################################################################## +## RELEASE +######################################################################## +## +## By default, mpm gets the latest versions of R2026b MathWorks products +## and support packages using this input file. +## +## To get a specific update for R2026b, set the desired update +## number (for example, 1, 2, 3, and so on) and uncomment the following +## line. An updateLevel of 0 gets the general release. + +# updateLevel=0 + + +######################################################################## +## INSTALLATION OR DOWNLOAD FOLDER +######################################################################## +## +## When installing products using 'mpm install': +## +## Specify the full path to the folder where you want to install the +## products. When adding products or support packages to an +## existing MATLAB installation, specify the full path to the folder +## where MATLAB is installed. +## +## Example: +## (Windows) destinationFolder=C:\Program Files\MATLAB\RXXXX +## (Linux) destinationFolder=/usr/local/RXXXX +## (macOS) destinationFolder=/Applications/MATLAB/RXXXX +## +## When downloading products using 'mpm download': +## +## Specify the full path to the folder where you want to download the +## products. For 'mpm download', this option is required. +## +## Example: +## (Windows) destinationFolder=C:\Users\\Downloads\MathWorks +## (Linux/macOS) destinationFolder=/home//downloads/mathworks +## +## Set the desired value for destinationFolder and +## uncomment the following line. + +# destinationFolder= + +######################################################################## +## SUPPORT PACKAGE DESTINATION +######################################################################## +## +## Specify the full path to the folder where you want to install +## support packages. If not specified, support packages are installed +## to the default support package location. +## +## Example: +## (Windows) supportPackageDestinationFolder=C:\shared\spkgs +## (Linux) supportPackageDestinationFolder=/home//spkgs +## (Mac) supportPackageDestinationFolder=/home//spkgs +## +## Set the desired value for supportPackageDestinationFolder and +## uncomment the following line. + +# supportPackageDestinationFolder= + +######################################################################## +## ACCEPT VENDOR LICENSE AGREEMENTS +######################################################################## +## +## To install support packages that contain vendor licenses, you must agree +## to accept these licenses. +## +## Uncomment the following line to accept all vendor license agreements. +## When you run mpm with this option, mpm displays the accepted agreements. + +# acceptVendorLicenses=yes + +######################################################################## +## PRODUCTS +######################################################################## +## +## Uncomment the lines for the products you want to install or download. + +#product.5G_Toolbox +#product.AUTOSAR_Blockset +#product.Aerospace_Blockset +#product.Aerospace_Toolbox +#product.Antenna_Toolbox +#product.Audio_Toolbox +#product.Automated_Driving_Toolbox +#product.Bioinformatics_Toolbox +#product.Bluetooth_Toolbox +#product.C2000_Microcontroller_Blockset +#product.Communications_Toolbox +#product.Computer_Vision_Toolbox +#product.Control_System_Toolbox +#product.Curve_Fitting_Toolbox +#product.DDS_Blockset +#product.DSP_HDL_Toolbox +#product.DSP_System_Toolbox +#product.Data_Acquisition_Toolbox +#product.Database_Toolbox +#product.Datafeed_Toolbox +#product.Deep_Learning_HDL_Toolbox +#product.Deep_Learning_Toolbox +#product.Econometrics_Toolbox +#product.Embedded_Coder +#product.Financial_Instruments_Toolbox +#product.Financial_Toolbox +#product.Fixed-Point_Designer +#product.Fuzzy_Logic_Toolbox +#product.GPU_Coder +#product.Global_Optimization_Toolbox +#product.HDL_Coder +#product.HDL_Verifier +#product.Image_Acquisition_Toolbox +#product.Image_Processing_Toolbox +#product.Industrial_Communication_Toolbox +#product.Instrument_Control_Toolbox +#product.LTE_Toolbox +#product.MATLAB +#product.MATLAB_Coder +#product.MATLAB_Compiler +#product.MATLAB_Compiler_SDK +#product.MATLAB_Copilot +#product.MATLAB_Parallel_Server +#product.MATLAB_Production_Server +#product.MATLAB_Report_Generator +#product.MATLAB_Test +#product.MATLAB_Web_App_Server +#product.Mapping_Toolbox +#product.Medical_Imaging_Toolbox +#product.Mixed-Signal_Blockset +#product.Model_Predictive_Control_Toolbox +#product.Model-Based_Calibration_Toolbox +#product.Motor_Control_Blockset +#product.Navigation_Toolbox +#product.Optimization_Toolbox +#product.Parallel_Computing_Toolbox +#product.Partial_Differential_Equation_Toolbox +#product.Phased_Array_System_Toolbox +#product.Point_Cloud_Toolbox +#product.Polyspace_Bug_Finder +#product.Polyspace_Bug_Finder_Server +#product.Polyspace_Code_Prover +#product.Polyspace_Code_Prover_Server +#product.Polyspace_Test +#product.Polyspace_Test_Server +#product.Polyspace_as_You_Code +#product.Powertrain_Blockset +#product.Predictive_Maintenance_Toolbox +#product.RF_Blockset +#product.RF_PCB_Toolbox +#product.RF_Toolbox +#product.ROS_Toolbox +#product.Radar_Toolbox +#product.Raspberry_Pi_Blockset +#product.Reinforcement_Learning_Toolbox +#product.Requirements_Toolbox +#product.Risk_Management_Toolbox +#product.RoadRunner +#product.Robotics_System_Toolbox +#product.Robust_Control_Toolbox +#product.STM32_Microcontroller_Blockset +#product.Satellite_Communications_Toolbox +#product.Sensor_Fusion_and_Tracking_Toolbox +#product.SerDes_Toolbox +#product.Signal_Integrity_Toolbox +#product.Signal_Processing_Toolbox +#product.SimBiology +#product.SimEvents +#product.Simscape +#product.Simscape_Battery +#product.Simscape_Driveline +#product.Simscape_Electrical +#product.Simscape_Fluids +#product.Simscape_Multibody +#product.Simulink +#product.Simulink_3D_Animation +#product.Simulink_Check +#product.Simulink_Coder +#product.Simulink_Compiler +#product.Simulink_Control_Design +#product.Simulink_Copilot +#product.Simulink_Coverage +#product.Simulink_Design_Optimization +#product.Simulink_Design_Verifier +#product.Simulink_Desktop_Real-Time +#product.Simulink_FMU_Builder +#product.Simulink_Fault_Analyzer +#product.Simulink_PLC_Coder +#product.Simulink_Real-Time +#product.Simulink_Report_Generator +#product.Simulink_Test +#product.Simulink_Variant_Manager +#product.SoC_Blockset +#product.Spreadsheet_Link +#product.Stateflow +#product.Statistics_and_Machine_Learning_Toolbox +#product.Symbolic_Math_Toolbox +#product.System_Composer +#product.System_Identification_Toolbox +#product.Text_Analytics_Toolbox +#product.UAV_Toolbox +#product.Vehicle_Dynamics_Blockset +#product.Vehicle_Network_Toolbox +#product.Vision_HDL_Toolbox +#product.Visual_Inspection_Toolbox +#product.WLAN_Toolbox +#product.Wavelet_Toolbox +#product.Wireless_HDL_Toolbox +#product.Wireless_Network_Toolbox +#product.Wireless_Testbench + + +######################################################################## +## SUPPORT PACKAGES +######################################################################## +## +## Uncomment the lines for the support packages you want to install or download. + +#product.3D_Asset_Processing_Library_for_MATLAB +#product.6G_Exploration_Library_for_5G_Toolbox +#product.AI_Verification_Library_for_Deep_Learning_Toolbox +#product.ASIC_Testbench_for_HDL_Verifier +#product.Aerospace_Blockset_Interface_for_Unreal_Engine_Projects +#product.Aerospace_Blockset_Maritime_Applications_Library +#product.Audio_Toolbox_Interface_for_SpeechBrain_and_Torchaudio_Libraries +#product.Automated_Driving_Toolbox_Importer_for_Zenrin_Japan_Map_API_3.0_(Itsumo_NAVI_API_3.0)_Service +#product.Automated_Driving_Toolbox_Interface_for_Eclipse_SUMO_Traffic_Simulator +#product.Automated_Driving_Toolbox_Interface_for_Unreal_Engine_Projects +#product.Automated_Driving_Toolbox_Model_for_Lidar_Lane_Detection +#product.Automated_Driving_Toolbox_Test_Suite_for_Euro_NCAP_Protocols +#product.CI/CD_Automation_for_Simulink_Check +#product.Communications_Toolbox_Support_Package_for_Analog_Devices_ADALM-Pluto_Radio +#product.Communications_Toolbox_Support_Package_for_RTL-SDR_Radio +#product.Communications_Toolbox_Support_Package_for_USRP_Radio +#product.Component_Deployment_Guideline_for_Embedded_Coder +#product.Computer_Vision_Toolbox_Interface_for_Nerfstudio_Library +#product.Computer_Vision_Toolbox_Interface_for_OpenCV_in_MATLAB +#product.Computer_Vision_Toolbox_Interface_for_OpenCV_in_Simulink +#product.Computer_Vision_Toolbox_Model_for_Apple_DepthPro_Network +#product.Computer_Vision_Toolbox_Model_for_BiSeNet_v2_Semantic_Segmentation_Network +#product.Computer_Vision_Toolbox_Model_for_Grounding_DINO_Object_Detection +#product.Computer_Vision_Toolbox_Model_for_Inflated-3D_Video_Classification +#product.Computer_Vision_Toolbox_Model_for_MapAnything_Network +#product.Computer_Vision_Toolbox_Model_for_Mask_R-CNN_Instance_Segmentation +#product.Computer_Vision_Toolbox_Model_for_Object_Keypoint_Detection +#product.Computer_Vision_Toolbox_Model_for_OpenAI_CLIP_Network +#product.Computer_Vision_Toolbox_Model_for_Pose_Mask_R-CNN_6-DoF_Object_Pose_Estimation +#product.Computer_Vision_Toolbox_Model_for_R(2+1)D_Video_Classification +#product.Computer_Vision_Toolbox_Model_for_RAFT_Optical_Flow_Estimation +#product.Computer_Vision_Toolbox_Model_for_RTMDet_Object_Detection +#product.Computer_Vision_Toolbox_Model_for_RetinaFace_Face_Detection +#product.Computer_Vision_Toolbox_Model_for_SOLOv2_Instance_Segmentation +#product.Computer_Vision_Toolbox_Model_for_SlowFast_Video_Classification +#product.Computer_Vision_Toolbox_Model_for_Text_Detection +#product.Computer_Vision_Toolbox_Model_for_Vision_Transformer_Network +#product.Computer_Vision_Toolbox_Model_for_YOLO_v2_Object_Detection +#product.Computer_Vision_Toolbox_Model_for_YOLO_v3_Object_Detection +#product.Computer_Vision_Toolbox_Model_for_YOLO_v4_Object_Detection +#product.Computer_Vision_Toolbox_Model_for_moondream_Vision_Language_Model +#product.Computer_Vision_Toolbox_OCR_Language_Data +#product.Data_Acquisition_Toolbox_Support_Package_for_Analog_Devices_ADALM1000_Hardware +#product.Data_Acquisition_Toolbox_Support_Package_for_Digilent_Analog_Discovery_Hardware +#product.Data_Acquisition_Toolbox_Support_Package_for_Measurement_Computing_Hardware +#product.Data_Acquisition_Toolbox_Support_Package_for_National_Instruments_NI-DAQmx_Devices +#product.Data_Acquisition_Toolbox_Support_Package_for_Windows_Sound_Cards +#product.Database_Toolbox_Interface_for_Neo4j_Bolt_Protocol +#product.Databricks_ODBC_Driver_for_Database_Toolbox +#product.Deep_Learning_Toolbox_Converter_for_ONNX_Model_Format +#product.Deep_Learning_Toolbox_Converter_for_PyTorch_Model_Format +#product.Deep_Learning_Toolbox_Converter_for_TensorFlow_models +#product.Deep_Learning_Toolbox_Importer_for_Caffe_Models +#product.Deep_Learning_Toolbox_Interface_for_LiteRT_Library +#product.Deep_Learning_Toolbox_Interface_for_alpha-beta-CROWN_Verifier +#product.Deep_Learning_Toolbox_Model_Compression_Library +#product.Deep_Learning_Toolbox_Model_for_AlexNet_Network +#product.Deep_Learning_Toolbox_Model_for_DarkNet-19_Network +#product.Deep_Learning_Toolbox_Model_for_DarkNet-53_Network +#product.Deep_Learning_Toolbox_Model_for_DenseNet-201_Network +#product.Deep_Learning_Toolbox_Model_for_EfficientNet-b0_Network +#product.Deep_Learning_Toolbox_Model_for_GoogLeNet_Network +#product.Deep_Learning_Toolbox_Model_for_Inception-ResNet-v2_Network +#product.Deep_Learning_Toolbox_Model_for_Inception-v3_Network +#product.Deep_Learning_Toolbox_Model_for_MobileNet-v2_Network +#product.Deep_Learning_Toolbox_Model_for_NASNet-Large_Network +#product.Deep_Learning_Toolbox_Model_for_NASNet-Mobile_Network +#product.Deep_Learning_Toolbox_Model_for_Places365-GoogLeNet_Network +#product.Deep_Learning_Toolbox_Model_for_ResNet-101_Network +#product.Deep_Learning_Toolbox_Model_for_ResNet-18_Network +#product.Deep_Learning_Toolbox_Model_for_ResNet-50_Network +#product.Deep_Learning_Toolbox_Model_for_ShuffleNet_Network +#product.Deep_Learning_Toolbox_Model_for_VGG-16_Network +#product.Deep_Learning_Toolbox_Model_for_VGG-19_Network +#product.Deep_Learning_Toolbox_Model_for_Xception_Network +#product.Embedded_Coder_Interface_to_QEMU_Emulator +#product.Embedded_Coder_Support_Package_for_ARM_Cortex-A_Processors +#product.Embedded_Coder_Support_Package_for_ARM_Cortex-M_Processors +#product.Embedded_Coder_Support_Package_for_ARM_Cortex-R_Processors +#product.Embedded_Coder_Support_Package_for_BeagleBone_Black_Hardware +#product.Embedded_Coder_Support_Package_for_Infineon_AURIX_TC3x_Microcontrollers +#product.Embedded_Coder_Support_Package_for_Infineon_AURIX_TC4x_Microcontrollers +#product.Embedded_Coder_Support_Package_for_Qualcomm_Hexagon_Processors +#product.Embedded_Coder_Support_Package_for_Real-Time_Linux +#product.Embedded_Coder_Support_Package_for_Renesas_RA_Microcontrollers +#product.Embedded_Coder_Support_Package_for_Renesas_RH850_Microcontrollers +#product.Embedded_Coder_Support_Package_for_Service-Oriented_Applications_on_Linux +#product.Ephemeris_Data_for_Aerospace_Toolbox +#product.Extended_Tire_Features_for_Vehicle_Dynamics_Blockset +#product.GPU_Coder_Interface_for_Deep_Learning +#product.GUIDE_to_App_Designer_Migration_Tool_for_MATLAB +#product.Geoid_Data_for_Aerospace_Toolbox +#product.HDL_Coder_Support_Package_for_Intel_FPGA_and_SoC_Devices +#product.HDL_Coder_Support_Package_for_Microchip_FPGA_and_SoC_Devices +#product.Hyperspectral_Imaging_Library_for_Image_Processing_Toolbox +#product.Image_Acquisition_Toolbox_Support_Package_for_DCAM_Hardware +#product.Image_Acquisition_Toolbox_Support_Package_for_GenICam_Interface +#product.Image_Acquisition_Toolbox_Support_Package_for_GigE_Vision_Hardware +#product.Image_Acquisition_Toolbox_Support_Package_for_Kinect_for_Windows_Sensor +#product.Image_Acquisition_Toolbox_Support_Package_for_Matrox_Hardware +#product.Image_Acquisition_Toolbox_Support_Package_for_National_Instruments_Frame_Grabbers +#product.Image_Acquisition_Toolbox_Support_Package_for_OS_Generic_Video_Interface +#product.Image_Acquisition_Toolbox_Support_Package_for_Point_Grey_Hardware +#product.Image_Acquisition_Toolbox_Support_Package_for_Teledyne_DALSA_Sapera_Hardware +#product.Image_Processing_Toolbox_Image_Data +#product.Image_Processing_Toolbox_Model_for_Circle_Detection +#product.Image_Processing_Toolbox_Model_for_Segment_Anything_Model +#product.Image_Processing_Toolbox_Model_for_Segment_Anything_Model_2 +#product.Instrument_Control_Toolbox_Support_Package_for_IVI_and_VXIplug&play_Drivers +#product.Instrument_Control_Toolbox_Support_Package_for_Keysight_(Agilent)_IO_Libraries_and_VISA_Interface +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-845x_I2C/SPI_Interface +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-DCPower_Power_Supplies +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-DMM_Digital_Multimeters +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-FGEN_Function_Generators +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-SCOPE_Oscilloscopes +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-SWITCH_Hardware +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_VISA_and_ICP_Interfaces +#product.Instrument_Control_Toolbox_Support_Package_for_Rohde_Schwarz_VISA_Interface +#product.Instrument_Control_Toolbox_Support_Package_for_Total_Phase_Aardvark_I2C/SPI_Interface +#product.Integro-Differential_Modeling_Framework_for_MATLAB +#product.Large-Scale_Cloud_Simulation_for_Simulink +#product.MATLAB_Basemap_Data_-_bluegreen +#product.MATLAB_Basemap_Data_-_colorterrain +#product.MATLAB_Basemap_Data_-_grayland +#product.MATLAB_Basemap_Data_-_grayterrain +#product.MATLAB_Basemap_Data_-_landcover +#product.MATLAB_Client_for_MATLAB_Production_Server +#product.MATLAB_Coder_Interface_for_Deep_Learning +#product.MATLAB_Coder_Interface_for_Visual_Studio_Code_Debugging +#product.MATLAB_Coder_Support_Package_for_NVIDIA_Jetson_and_NVIDIA_DRIVE_Platforms +#product.MATLAB_Coder_Support_Package_for_PyTorch_and_LiteRT_Models +#product.MATLAB_Object_Performance_Improvement +#product.MATLAB_Support_Package_for_Android_Sensors +#product.MATLAB_Support_Package_for_Apple_iOS_Sensors +#product.MATLAB_Support_Package_for_Arduino_Hardware +#product.MATLAB_Support_Package_for_BeagleBone_Black_Hardware +#product.MATLAB_Support_Package_for_IP_Cameras +#product.MATLAB_Support_Package_for_LEGO_MINDSTORMS_EV3_Hardware +#product.MATLAB_Support_Package_for_Parrot_Drones +#product.MATLAB_Support_Package_for_Quantum_Computing +#product.MATLAB_Support_Package_for_Ryze_Tello_Drones +#product.MATLAB_Support_Package_for_USB_Webcams +#product.MATLAB_Support_for_HEIF/HEIC_Image_Format +#product.MATLAB_Support_for_MinGW-w64_C/C++/Fortran_Compiler +#product.MATLAB_Support_for_OpenJDK +#product.MATLAB_Support_for_Python_3.13 +#product.Machine_Learning_Models_for_Antenna_Toolbox +#product.MariaDB_ODBC_Driver_for_Database_Toolbox +#product.Medical_Imaging_Toolbox_Interface_for_Cellpose +#product.Medical_Imaging_Toolbox_Interface_for_MONAI_Label_Library +#product.Medical_Imaging_Toolbox_Interface_for_Whole_Slide_Imaging_File_Reader +#product.Medical_Imaging_Toolbox_Model_for_Medical_Segment_Anything_Model +#product.Mixed-Signal_Blockset_Models +#product.Modelscape_for_MATLAB +#product.Multi-Version_Co-Simulation_for_Simulink +#product.Optical_Design_and_Simulation_Library_for_Image_Processing_Toolbox +#product.Point_Cloud_Toolbox_Interface_for_OpenPCDet_Library +#product.Point_Cloud_Toolbox_Model_for_PointNet++_Classification +#product.Point_Cloud_Toolbox_Model_for_RandLA-Net_Semantic_Segmentation +#product.Point_Cloud_Toolbox_Model_for_SNAP_Segmentation_Network +#product.Point_Cloud_Toolbox_Support_Package_for_Hokuyo_LiDAR_Sensors +#product.Point_Cloud_Toolbox_Support_Package_for_Ouster_Lidar_Sensors +#product.Point_Cloud_Toolbox_Support_Package_for_SICK_Lidar_Sensors +#product.Point_Cloud_Toolbox_Support_Package_for_Velodyne_LiDAR_Sensors +#product.PostgreSQL_ODBC_Driver_for_Database_Toolbox +#product.Power_Converter_Control_with_Motor_Control_Blockset +#product.Powertrain_Blockset_Drive_Cycle_Data +#product.RF_Blockset_Models_for_Analog_Devices_RF_Transceivers +#product.ROS_Toolbox_Support_Package_for_TurtleBot-Based_Robots +#product.Radar_Toolbox_Support_Package_for_Texas_Instruments_mmWave_Radar_Sensors +#product.Reduced_Order_Modeler_for_MATLAB +#product.Robotics_System_Toolbox_Interface_for_Unreal_Engine_Projects +#product.Robotics_System_Toolbox_Offroad_Autonomy_Library +#product.Robotics_System_Toolbox_Robot_Library_Data +#product.Robotics_System_Toolbox_Support_Package_for_Kinova_Gen3_Manipulators +#product.Robotics_System_Toolbox_Support_Package_for_Universal_Robots_UR_Series_Manipulators +#product.Scenario_Builder_for_Automated_Driving_Toolbox +#product.SerDes_Toolbox_Support_Package_for_Linux_Cross_Compiler +#product.Simulink_3D_Animation_Interface_for_Unreal_Engine_Projects +#product.Simulink_Coder_Support_Package_for_ARM_Cortex-based_VEX_Microcontroller +#product.Simulink_Coder_Support_Package_for_BBC_microbit_Board +#product.Simulink_Coder_Support_Package_for_BeagleBone_Blue_Hardware +#product.Simulink_Coder_Support_Package_for_NXP_FRDM-K64F_Board +#product.Simulink_Coder_Support_Package_for_NXP_FRDM-KL25Z_Board +#product.Simulink_Coder_Support_Package_for_VEX_EDR_V5_Robot_Brain +#product.Simulink_Interface_for_Siemens_MF-Tyre/MF-Swift_Tire_Model +#product.Simulink_Real-Time_XIL_Support_Package +#product.Simulink_Support_Package_for_Android_Devices +#product.Simulink_Support_Package_for_Arduino_Hardware +#product.Simulink_Support_Package_for_LEGO_MINDSTORMS_EV3_Hardware +#product.Simulink_Support_Package_for_Parrot_Minidrones +#product.Simulink_Support_Package_for_Rust_Code +#product.Simulink_Test_Support_Package_for_ASAM_XIL +#product.SoC_Blockset_Support_Package_for_AMD_FPGA_and_SoC_Devices +#product.Software_Component_Designer_for_Simulink +#product.Streaming_Data_Framework_for_MATLAB_Production_Server +#product.Text_Analytics_Toolbox_Model_for_BERT-Base_Multilingual_Cased_Network +#product.Text_Analytics_Toolbox_Model_for_BERT-Base_Network +#product.Text_Analytics_Toolbox_Model_for_BERT-Large_Network +#product.Text_Analytics_Toolbox_Model_for_BERT-Mini_Network +#product.Text_Analytics_Toolbox_Model_for_BERT-Small_Network +#product.Text_Analytics_Toolbox_Model_for_BERT-Tiny_Network +#product.Text_Analytics_Toolbox_Model_for_all-MiniLM-L12-v2_Network +#product.Text_Analytics_Toolbox_Model_for_all-MiniLM-L6-v2_Network +#product.Text_Analytics_Toolbox_Model_for_fastText_English_16_Billion_Token_Word_Embedding +#product.Text_Analytics_Toolbox_Model_from_UDify_Data +#product.Third-Party_Tool_Connector_for_Requirements_Toolbox +#product.Time_Series_Anomaly_Detection_for_MATLAB +#product.UAV_Toolbox_Interface_for_Unreal_Engine_Projects +#product.UAV_Toolbox_Support_Package_for_ArduPilot_Autopilots +#product.UAV_Toolbox_Support_Package_for_PX4_Autopilots +#product.Vehicle_Dynamics_Blockset_Interface_for_Unreal_Engine_Projects +#product.Vehicle_Dynamics_Blockset_Maneuver_Data +#product.Visual_Inspection_Toolbox_Model_for_CounTR_Object_Counting +#product.Visual_Inspection_Toolbox_Model_for_Student-Teacher_Anomaly_Detection +#product.Visual_Inspection_Toolbox_Model_for_YOLOX_Object_Detection +#product.WINNER_II_Channel_Model_for_Communications_Toolbox +#product.Wireless_Testbench_Support_Package_for_NI_USRP_Radios + + +######################################################################## +## OPTIONAL FEATURES +######################################################################## +## +## Uncomment the lines for the optional features you want to install or download. + +#product.3D_Scenarios +#product.Airport_Scene +#product.Apple_Hill_Scene +#product.Construction_Site_Scene +#product.Curved_Road_Scene +#product.Double_Lane_Change_Scene +#product.Earth_Scene +#product.Empty_Grass_Scene +#product.Empty_Scene +#product.Griffiss_Airport_Scene +#product.Large_Parking_Lot_Scene +#product.Moon_Scene +#product.Multi-Sensor_Calibration_Tools +#product.NVIDIA_TensorRT +#product.Offroad_Pit_Mining_Scene +#product.Open_Surface_Scene +#product.Parking_Lot_Scene +#product.Room_Acoustics_Simulation +#product.Simscape_Acoustics +#product.Space_Scene +#product.Straight_Road_Scene +#product.Suburban_Scene +#product.US_City_Block_Scene +#product.US_Highway_Scene +#product.Underwater_Acoustic_Channels +#product.Underwater_Scene +#product.Virtual_MCity_Scene +#product.ZalaZONE_Automotive_Proving_Ground_High-speed_Handling_Course_Scene +#product.ZalaZONE_Automotive_Proving_Ground_Hill_Tracks_Scene +#product.ZalaZONE_Automotive_Proving_Ground_Smart_City_Scene + + +######################################################################## +## CHECKSUM +######################################################################## +## +## NOTE: DO NOT edit this field. MathWorks uses this field to +## check the integrity of the input file. Changing the value +## of the checksum field invalidates this input file. + +?checksum=UjIwMjZi From 69f1eb43b90e740ea0795bd34ccdf31bb48da383 Mon Sep 17 00:00:00 2001 From: chiku Date: Tue, 13 Oct 2026 15:14:00 +0900 Subject: [PATCH 38/41] Sync R2025a/R2025b/R2026a mpm input files mathworks-ref-arch/matlab-dockerfile@f419020 added a commented-out supportPackageDestinationFolder section to these files. No product lines changed, so the generated product name lists are unaffected. Co-Authored-By: Claude Fable 5.1 --- .../settings/mpm_inputs/mpm_input_r2025a.txt | 18 ++++++++++++++++++ .../settings/mpm_inputs/mpm_input_r2025b.txt | 18 ++++++++++++++++++ .../settings/mpm_inputs/mpm_input_r2026a.txt | 18 ++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/addons/binderhub/settings/mpm_inputs/mpm_input_r2025a.txt b/addons/binderhub/settings/mpm_inputs/mpm_input_r2025a.txt index 3e012822ef1..7570f6f2b3e 100644 --- a/addons/binderhub/settings/mpm_inputs/mpm_input_r2025a.txt +++ b/addons/binderhub/settings/mpm_inputs/mpm_input_r2025a.txt @@ -71,6 +71,24 @@ # destinationFolder= +######################################################################## +## SUPPORT PACKAGE DESTINATION +######################################################################## +## +## Specify the full path to the folder where you want to install +## support packages. If not specified, support packages are installed +## to the default support package location. +## +## Example: +## (Windows) supportPackageDestinationFolder=C:\shared\spkgs +## (Linux) supportPackageDestinationFolder=/home//spkgs +## (Mac) supportPackageDestinationFolder=/home//spkgs +## +## Set the desired value for supportPackageDestinationFolder and +## uncomment the following line. + +# supportPackageDestinationFolder= + ######################################################################## ## ACCEPT VENDOR LICENSE AGREEMENTS ######################################################################## diff --git a/addons/binderhub/settings/mpm_inputs/mpm_input_r2025b.txt b/addons/binderhub/settings/mpm_inputs/mpm_input_r2025b.txt index 3ae7663149d..2340f6e82da 100644 --- a/addons/binderhub/settings/mpm_inputs/mpm_input_r2025b.txt +++ b/addons/binderhub/settings/mpm_inputs/mpm_input_r2025b.txt @@ -71,6 +71,24 @@ # destinationFolder= +######################################################################## +## SUPPORT PACKAGE DESTINATION +######################################################################## +## +## Specify the full path to the folder where you want to install +## support packages. If not specified, support packages are installed +## to the default support package location. +## +## Example: +## (Windows) supportPackageDestinationFolder=C:\shared\spkgs +## (Linux) supportPackageDestinationFolder=/home//spkgs +## (Mac) supportPackageDestinationFolder=/home//spkgs +## +## Set the desired value for supportPackageDestinationFolder and +## uncomment the following line. + +# supportPackageDestinationFolder= + ######################################################################## ## ACCEPT VENDOR LICENSE AGREEMENTS ######################################################################## diff --git a/addons/binderhub/settings/mpm_inputs/mpm_input_r2026a.txt b/addons/binderhub/settings/mpm_inputs/mpm_input_r2026a.txt index 96014b83127..7c8376a7f47 100644 --- a/addons/binderhub/settings/mpm_inputs/mpm_input_r2026a.txt +++ b/addons/binderhub/settings/mpm_inputs/mpm_input_r2026a.txt @@ -71,6 +71,24 @@ # destinationFolder= +######################################################################## +## SUPPORT PACKAGE DESTINATION +######################################################################## +## +## Specify the full path to the folder where you want to install +## support packages. If not specified, support packages are installed +## to the default support package location. +## +## Example: +## (Windows) supportPackageDestinationFolder=C:\shared\spkgs +## (Linux) supportPackageDestinationFolder=/home//spkgs +## (Mac) supportPackageDestinationFolder=/home//spkgs +## +## Set the desired value for supportPackageDestinationFolder and +## uncomment the following line. + +# supportPackageDestinationFolder= + ######################################################################## ## ACCEPT VENDOR LICENSE AGREEMENTS ######################################################################## From e92d12ff9f393a1882f9d0a745e054d5cd07edc2 Mon Sep 17 00:00:00 2001 From: hcphat Date: Fri, 16 Oct 2026 16:56:24 +0700 Subject: [PATCH 39/41] =?UTF-8?q?Ref:=20=E6=A9=9F=E9=96=A2=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E8=80=85=E6=A9=9F=E8=83=BDAPI=E3=81=AE=E9=81=A9?= =?UTF-8?q?=E5=88=87=E3=81=AA=E6=A8=A9=E9=99=90=E4=BB=98=E4=B8=8E:=20Updat?= =?UTF-8?q?e=20logic=20osf=5Fsuperuser=5Fcheck=20and=20handle=20admin/logi?= =?UTF-8?q?n/=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- admin/base/urls.py | 14 ++++++ admin/base/utils.py | 8 +++- admin_tests/base/test_urls.py | 59 ++++++++++++++++++++++++ admin_tests/base/test_utils.py | 38 ++++++++++++++- admin_tests/quota_recalc/test_views.py | 31 ++++++++++++- admin_tests/rdm_statistics/test_views.py | 19 ++++++++ 6 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 admin_tests/base/test_urls.py diff --git a/admin/base/urls.py b/admin/base/urls.py index 85946577196..d201a3e5b38 100644 --- a/admin/base/urls.py +++ b/admin/base/urls.py @@ -1,12 +1,26 @@ from django.conf.urls import include, url from django.contrib import admin +from django.http import Http404 from admin.base.settings import ADMIN_BASE, DEBUG, ENABLE_LOGIN_FORM from admin.base import views base_pattern = '^{}'.format(ADMIN_BASE) + +def admin_login_disabled(request): + # admin.site.urls provides its own password-based login (AdminSite.login), + # which must not exist while password-based login is disabled. + raise Http404 + admin_urls = [ url(r'^$', views.home, name='home'), +] + +if not ENABLE_LOGIN_FORM: + # must be registered before admin.site.urls, otherwise ^admin/ matches first + admin_urls.append(url(r'^admin/login/$', admin_login_disabled, name='admin_login_disabled')) + +admin_urls += [ url(r'^admin/', admin.site.urls), url(r'^asset_files/', include('admin.asset_files.urls', namespace='asset_files')), url(r'^banners/', include('admin.banners.urls', namespace='banners')), diff --git a/admin/base/utils.py b/admin/base/utils.py index 4f063bbbea3..ff86f9f7900 100644 --- a/admin/base/utils.py +++ b/admin/base/utils.py @@ -38,7 +38,13 @@ def osf_staff_check(user): def osf_superuser_check(user): - return user.is_authenticated and user.is_superuser + """An inactive or unregistered superuser is not an Integrated Admin.""" + return bool( + user.is_authenticated and + user.is_active and + user.is_registered and + user.is_superuser + ) def superuser_required(view_func): diff --git a/admin_tests/base/test_urls.py b/admin_tests/base/test_urls.py new file mode 100644 index 00000000000..c6c4ff60aa5 --- /dev/null +++ b/admin_tests/base/test_urls.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +import mock +from django.urls import clear_url_caches, resolve +from django.http import Http404 +from django.test import RequestFactory +from django.utils.six.moves import reload_module +from nose import tools as nt + +from admin.base import urls as admin_urls_module +from tests.base import AdminTestCase + + +class TestAdminLoginUrl(AdminTestCase): + """/admin/login/ (AdminSite.login from admin.site.urls) is a second entry point + of password-based authentication, so it must follow ENABLE_LOGIN_FORM.""" + + def _reload_urls(self): + reload_module(admin_urls_module) + clear_url_caches() + + def _with_login_form(self, enabled): + patcher = mock.patch('admin.base.settings.ENABLE_LOGIN_FORM', enabled) + patcher.start() + self.addCleanup(self._restore_urls) + self.addCleanup(patcher.stop) + self._reload_urls() + + def _restore_urls(self): + # cleanups run LIFO: patcher.stop() has already run, so this reloads with the real setting + reload_module(admin_urls_module) + clear_url_caches() + + def _call_login_view(self, request): + match = resolve('/admin/login/') + return match.func(request, *match.args, **match.kwargs) + + def test_login_is_404_for_get_when_login_form_disabled(self): + self._with_login_form(False) + request = RequestFactory().get('/admin/login/') + with nt.assert_raises(Http404): + self._call_login_view(request) + + def test_login_is_404_for_post_when_login_form_disabled(self): + self._with_login_form(False) + request = RequestFactory().post('/admin/login/', {'username': 'user@example.com', 'password': 'secret'}) + with nt.assert_raises(Http404): + self._call_login_view(request) + + def test_login_route_is_disabled_view_when_login_form_disabled(self): + self._with_login_form(False) + nt.assert_equal(resolve('/admin/login/').url_name, 'admin_login_disabled') + + def test_login_route_is_admin_site_login_when_login_form_enabled(self): + self._with_login_form(True) + nt.assert_equal(resolve('/admin/login/').view_name, 'admin:login') + + def test_admin_site_urls_other(self): + self._with_login_form(False) + nt.assert_equal(resolve('/admin/').view_name, 'admin:index') diff --git a/admin_tests/base/test_utils.py b/admin_tests/base/test_utils.py index 37738383542..910b4f02e63 100644 --- a/admin_tests/base/test_utils.py +++ b/admin_tests/base/test_utils.py @@ -17,7 +17,7 @@ from osf.models import Subject, OSFUser, Collection from osf.models.provider import rules_to_subjects -from admin.base.utils import get_subject_rules, change_embargo_date +from admin.base.utils import get_subject_rules, change_embargo_date, osf_superuser_check from osf.admin import OSFUserAdmin @@ -255,3 +255,39 @@ def test_from_json(self): def test_from_json__file_not_found(self): with pytest.raises(Exception): from_json('file-info-schema2.json') + + +class TestOsfSuperuserCheck(AdminTestCase): + """osf_superuser_check must have the same condition as RdmPermissionMixin.is_super_admin.""" + + def setUp(self): + super(TestOsfSuperuserCheck, self).setUp() + self.user = UserFactory() + self.user.is_superuser = True + self.user.save() + + def _update(self, **fields): + # queryset.update() skips OSFUser.save(), which would otherwise recompute is_active + OSFUser.objects.filter(pk=self.user.pk).update(**fields) + self.user.refresh_from_db() + + def test_valid_superuser(self): + assert_true(osf_superuser_check(self.user)) + + def test_non_superuser(self): + self._update(is_superuser=False) + assert_false(osf_superuser_check(self.user)) + + def test_inactive_superuser(self): + self._update(is_active=False) + assert_true(self.user.is_superuser) + assert_false(osf_superuser_check(self.user)) + + def test_unregistered_superuser(self): + self._update(is_registered=False) + assert_true(self.user.is_superuser) + assert_false(osf_superuser_check(self.user)) + + def test_anonymous(self): + from django.contrib.auth.models import AnonymousUser + assert_false(osf_superuser_check(AnonymousUser())) diff --git a/admin_tests/quota_recalc/test_views.py b/admin_tests/quota_recalc/test_views.py index 1263d179b26..59b885a8432 100644 --- a/admin_tests/quota_recalc/test_views.py +++ b/admin_tests/quota_recalc/test_views.py @@ -8,7 +8,7 @@ from admin.quota_recalc import views from api.base import settings as api_settings -from osf.models import UserQuota +from osf.models import OSFUser, UserQuota from osf_tests.factories import AuthUserFactory, InstitutionFactory, RegionFactory from tests.base import AdminTestCase @@ -148,6 +148,35 @@ def test_user_allowed_for_superuser(self, mock_usedquota): response = views.user(request, guid=self.target_user._id) nt.assert_equal(response.status_code, 200) + def _invalid_superuser(self, **fields): + # queryset.update() skips OSFUser.save(), which would otherwise recompute is_active + OSFUser.objects.filter(pk=self.superuser.pk).update(**fields) + self.superuser.refresh_from_db() + nt.assert_true(self.superuser.is_superuser) + return self.superuser + + def _assert_denied(self, view, user, **kwargs): + request = RequestFactory().get('/fake_path') + request.user = user + with nt.assert_raises(PermissionDenied): + view(request, **kwargs) + + def test_all_users_denied_for_inactive_superuser(self): + user = self._invalid_superuser(is_active=False) + self._assert_denied(views.all_users, user) + + def test_all_users_denied_for_unregistered_superuser(self): + user = self._invalid_superuser(is_registered=False) + self._assert_denied(views.all_users, user) + + def test_user_denied_for_inactive_superuser(self): + user = self._invalid_superuser(is_active=False) + self._assert_denied(views.user, user, guid=self.target_user._id) + + def test_user_denied_for_unregistered_superuser(self): + user = self._invalid_superuser(is_registered=False) + self._assert_denied(views.user, user, guid=self.target_user._id) + class TestCalculateQuota(AdminTestCase): diff --git a/admin_tests/rdm_statistics/test_views.py b/admin_tests/rdm_statistics/test_views.py index 181c64d51ce..5f60160a58f 100644 --- a/admin_tests/rdm_statistics/test_views.py +++ b/admin_tests/rdm_statistics/test_views.py @@ -542,6 +542,25 @@ def test_send_stat_mail_allowed_for_superuser(self): response = views.send_stat_mail(request) nt.assert_equal(response.status_code, 200) + def test_send_stat_mail_denied_for_inactive_superuser(self): + # queryset.update() skips OSFUser.save(), which would otherwise recompute is_active + OSFUser.objects.filter(pk=self.superuser.pk).update(is_active=False) + self.superuser.refresh_from_db() + nt.assert_true(self.superuser.is_superuser) + request = RequestFactory().get('/fake_path') + request.user = self.superuser + with nt.assert_raises(PermissionDenied): + views.send_stat_mail(request) + + def test_send_stat_mail_denied_for_unregistered_superuser(self): + OSFUser.objects.filter(pk=self.superuser.pk).update(is_registered=False) + self.superuser.refresh_from_db() + nt.assert_true(self.superuser.is_superuser) + request = RequestFactory().get('/fake_path') + request.user = self.superuser + with nt.assert_raises(PermissionDenied): + views.send_stat_mail(request) + class TestIndexViewPermission(AdminTestCase): """IndexView must be restricted to Integrated Admin or Institutional Admin.""" From 10e474d97a72420b160b6f799b8a75a23da0d5cf Mon Sep 17 00:00:00 2001 From: Yusaku Kitabatake Date: Tue, 20 Oct 2026 08:42:26 +0900 Subject: [PATCH 40/41] Add eduGAIN logo image --- .../img/institutions/banners/edugain-logo.png | Bin 0 -> 9985 bytes .../edugain-shield-rounded-corners.png | Bin 0 -> 4454 bytes .../img/institutions/shields/edugain-shield.png | Bin 0 -> 4454 bytes 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 website/static/img/institutions/banners/edugain-logo.png create mode 100644 website/static/img/institutions/shields-rounded-corners/edugain-shield-rounded-corners.png create mode 100644 website/static/img/institutions/shields/edugain-shield.png diff --git a/website/static/img/institutions/banners/edugain-logo.png b/website/static/img/institutions/banners/edugain-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..812f470d2104bd2d55095609da38a961eeeef55b GIT binary patch literal 9985 zcmX|nby!tT)b1$`-6?tKkS;+&KpF(38>Er$?vRiU2|<*Q20^+G-Q5k+-Cg(mzWd#~ z{@8mzd(Sg_X3dIu-&v6=N-~&eq-X#D08>s@QWXFIfe_D5DF0oVu=JM^w^y!G+OBF2 z7OozjoXr8^W)3FiFgd$VmgcJFpUk|R`pty_0ES#SNpW@0g+l{WPXdkF<9u(q#-z${ zNMc`Nju;M}nNe6mP~TUwTeuT}lcQIxZ7>Q7N-Qhl8cU0|D*y(uZ-5l%Qm$gCQ6&p-LSS#odm%k>|^ikQnl-n=Zdz1HGOa&pNDDr`qZzxxn)-Kke zQgMH9O?M_OgApzOxO==^#4e+Br095mPN9Np=zWF#$kA zLdp+;I!182_QP z!O`1<=%V+?m+<1_b*61AJPW#Mm-xSm7Q>h;Rv5N)XMRvhl{#xM zKp$anSnhV;pxNpC-xfKP;T4)pJ~5w;ciH-OGF^+?gCV9{sMfWa-vlNAHBctWFS(zJ z%c_#?(scEPTh%u8-%4fo`D5nqGkMuYu`(zTWlHE3sz!)|YxVLQ<1CBsMe(@81H9O5 z*plH}rQ?GeKI6T?C_C)Q90o3Z2bISLro_XI-wexpD#_eZA5VtvR`n{(Oi2s)aYE={ z9(tB?WMB2%(_{=8C8FJlVLP$r}nQ*%N@UN=VM?wPoiu zvDbQF!RlRb#`2)+Nz9v_qw5#$el6sSI*ruOb=ptD+dBi-+`q=Jbz44-_UvO@;wI!R z=Tr#RwwZLlp0yOQyWcB~D4}Hl7O5&^Nf3a3+B<9r!vLyb2`C72g@K=D`eoa`h{-e6 z6~LO+OjY3D>w4qfM5jUb9j3RI^hzjsvV^v?ww#sJ?Q|{%mwp$E>f78qP&VhbT8aUx z10`3SI8!tUK-RrEzoQwp#j5#jea4L}g6>#*M=>EnJshwYrc?A-)LdhtT~VD<%1g(t zLdHDYk^bkIsH%(*BhqSR?SKaMKOz4AaQgAA^Q7m?e#>Tjlk44bRw2#7>Deb4L{6swUE`v%|=Y z_eQ!=96`Hvi+M^9PSbn;(m}OR(Rl?}dcBue@?ZktT#Fkrw z-G!mjEetz753^kcT>7^~N3-GD>uT*w8DV|_596VGQP(#AJDF)2Vba2CXdidjrUym> z!pm`xc{zjB3A^(mrsVp+zho(Y`LRrq&ATQO0&nUVab|awC7i{ahe}RHhQj^^#6eQ@uRa9pG|K@Fi;v5o%!UFBvHtZsNAQH%zEL1thSsHBC>KmF zXNNv7gSW~|$l7SVjm6|IRSQJK>4}jXdYB@XM|Tn^>Ek}ce!q>qZZIfrSIZDWHsd;j z!FZGS;Upye;$nK`X6W{SyKUTm-PtJiA!~(H*EsQ$a5ny1gbs3lmq(J{G+($Zv1mS= zR+pvP#(4juw~_MN2w*Ka48>An=Pbua-&qL(SkEb{0&+)v9#fLoNJYKJR_%KY3 z^Jv^y_evFYgd{7Wo+=w1a~xgQlY}BRGrw{Ec%RZL{$9CP*3z+0E;)Hsw(mnV`7?UM zsD|-|?=G!N>74(lD>vwMZ-2Q&-Isj%ve4k*Zw}7pk<2!X2bY0M@?Nrz7Cdj`U(78$~{##6K){xSCd+e96 zTxlrSaYi`ePmS7Cxu z3vX0YA+uOJnc_Q~+g0&cAzz{IKmrLf92Bh)6aww9o2r1Le2n*5r-#Rteo<@kcp|bA zq}?#c9$T*Fh>n0j*v}{O`_=x( zNneNNVni%d^^w9Mg*Q0ONzNtzOw&M2y6>lQgauB04$!A;q|(O!$e(yIW12R5N?4fq zOpNyo^@gd#9J14JMkC4q;Ox=aO^p;J++KS}-@E}Hnlsj5erd5EH7 zTFd*T!AMpy1fC8pHNU%OQyaRI$<(PJJxa2nIrhWL&t6)lyPNRNeVh@hh`tmNenIkI z<@|v2Yi*f}<~z}eVQu2o&%Z=xZ9ec|bV1%R$_Wm-0#my0n4t8@-}`mKb|tPO9u~*D zgd(O*Y8t56W1{D!H2MAPz{xw|(pO9V{sOy4A%-k@2TrVCN!B}NBxR5tkzqMvRM zig+oYxn43|Grp{o3t&nECW28l+HjmlksSli3Q4x;%`ksh-ufCI1$kt|sbKr4K6L5I zyp|U>$u?k(u~K~tvdZCCn42+`q@VAFdW z7H^3qK?;Xnf6x(<+U@I+CGS=i{js`)QrM(N>baWSq~)}OL&E^TvJ$j0?>^w*_pf%I z)b+|Vj}m%|e$Ki-$olPuIy!2=#D_cUBo{FQWm1?^EoY=JQKI_wP$hkS?g!*-8*Lg9 z2_MMTp+^W z?JVh1tF9qHo6-m#=) zb`&gODjVR!_mq+;Djf3=2h(*6nIEZ2Sggy&#Sf-&l5lCx5Ux)zpnPbA{15IwaVTbEMXP#gpTb^l@V+D|`yQ)}@*cu3 zi@=tOQYqQ=RnX7}(P(TlrQd@XA2sV~tze-y2|?n4UvZ`=wSTA6sYlBbFZP;oISn-R zNq-qB)|e2q8VEmO%% z_?gR%NHB;gE93@NMx_hpKDn zN!Q6T$^B@1k=M92P4D7~xQnH^UFDBS`iGN)V-tSy-JPex7?6BS)vAKYifMcbj7!`; z`110}7FNphe9DzZ;CqH|)y4S+tJjdNDC6=GtJMOJMu|F6S0rz57I=tDv&arl$3-fO z*fA=UQF6V1eXfZvHxQqOa}jcbnELJM0#Kx^^%ubyUI34lb2NfdZ4d0zv~zk1#mJH{RJ?2k83{ z9AH%u5Nfh!T5iRN6p%KkLPBryN7UYu|) zyZ&K-r~YYF*u5S@F$5r}8j*cv))J8(+hWeP(4c@8trU7;3@peiylM zpFW`_SOfCdK2nj<&8=|AiPD!LlEwL)nIbDB4N$G<@_h4C9-P?NvC%g$(R@?NSOC;m zaLAktR4trIpDt_ci7>|r?z)3()+`)LI&#MB=O^Cy@-8A^4 zuCD5@@lJg`3j4o1e09IYhj*OPAO=k|o|quI{sWC%lR!m)x}~^rl^Fzj&I>7(^Wd}gKWp!s~mEosAM2lu(33ZiEa2& z^9${c&)YlLne3!ii*Nzgwim8E87egltG=&p`)QP^5qgL+Y(I4tZk7uE0!(irEvPhzX>sS+7-koN=Z36Ai8ta@kK6>9@1(Vvw}wh1Ww)KLAk-(sCP=Icu(f|Zd-LuUOBj>s!KppYI_rAR0ZW&{Xr)Ey>AHHB5YS81!RT5<|^x;a} zNZWUiM_hxkq9L#3F%3iDo~}IGs+m>pRqK$|YdY>F#)SsrKYKvMN=L6NnM{l?OZNen zKG^byiJIscoI!#fSuNX*J0SP-(1fV+q(|pb&d9XKR7c0)9Xk4EX0ZWaVk4VFGBm=t zgcWn8pH@WTm=wb^0o&~enKLtK*o#^T47e)X3+LGWBz?b9osH{d2ZwNtj#e(x@qJm= zsm8aYKBT=%X0HJjYh|sYs8wJqm@3hwd7WT^6tPk4lbwL&paSI zL3@mKVTn3y^6;NP5^ywTy8p;=`A;j6gI8^(mEGt(Vy2wrjt=;~K1|M5FA=y;b{b*Z zKlpuCI9a2Q3lCqyS3?hL2E;tA#I`&b6QUI=dC23HSSRuS<>vlnF}&YbsaKCb<27zb zWMjevARD`cj#bT@~uT-t4llN3tB)fX#W-Obf@NoIjp|{+yn=&h0GMEAu6LGb{a@tti zrlMgGO|+|DPFI&JfvW1JF2I^p0RE`Y4no}bhTqKAd3$BBuuW?kSdHkN zczp|`xsvkwlF;r<`|z6)CF^6-m}pNgk7GFo7fi${YcrJYyZ9y;b)-z$bDC^Ew#Os% zeQW%td$hdb>FOsOM<(hKNS`X-O^mwxao+Akzx{ZoOR7RC6q^_BEcTGl?#2ObJC*>k z;dITDTFDq=MljIdZ3V)Y>dCO~B=FXKW(Gl)MQ~uL)ZGuQ1NW#f0%Vm?egwDuQH6Kc zBy(G=Y@6u+H#{sTuhTNR9spi1z<5fci#$Z(fy%$S{S!=*&O$=$X`SFu(@r`gSsKUD zd%!Q?#oh2sVkf0;zkmST*IoIl3189j8Zyq-tg9MkjVeZs0C^|U1oX8;=E&nUpd;>h zg@&^^KYBPKi{K0Bszd29wJRw868UZXaoznvL9TLfhmRT3G=O0|DQva~`3@WzNu6Df%MuwE$A%$@c*Z zYs*-Diw4y`>63pDjB%Rt;}E28SS|evSma&k#dUBFkE;?5ONVpie1ZlJ`Rau6nH92B z3CG2JB?^H8`jBnpj_s;o9c?S5reU|K@+$bIluSIqD^D!PyHHWJ5y2U0Iun{B z`*#Sf&_e!2_OXQ%;%*_4q|B@nqa}d?PUm1g${n0K7y}YlWSDcVVFia$_H~hh0%Ceb zuW6hopq)Dv+R@I->;gNhcukTQ5dmoc3=lcy<1R5NT;Up#X#@}5vEo%1wMlV?gm##U z!vA*7{~89ifr(CVgKHkIj(HfFDH5GTf8mdbQs^BQ~10f|Cv$|v2`3)oVM4Fr~G zxwl_?dz*kZ_7s;#H2zhftd^M{6>IP%B+=ucWFo~o&x>o;pKU)N68x3mOa!;l zQ&;@UzFW;ePd?d$+V(t*Ti ztSlJa#xhj773TyRRQR7$j~sc5RTC$j6HE^K^zs>l@?{t*ll6-q56I8b0d_dwaRkj# zjChU-yco>a7(4j+yCtns#_RJy75_GE{?H|%-ftW2~Sd$%P#0j=P*a8!^ge< zyt?xH>h8i))5b@sxCTm42NQ#*aXB%A_G>(R#q_B!R*&EFeWVn~Y0lBh{HXX!5GV+h zMGq0|jl;cS8xf=lPC9sh`fmaR;WW7;l}Fw1Paui7G0vYmAvfqq{z!RnUX;*$;9xgf zDms|~B$O~A%1|LKR`~Fj`ZZ;4R=s+$`yl-_>C-Tu3!79hkGO>IB?4@~-kL2J;3KF5 z!k58U z3i$k^P8t0@n5boM$Y$o8cDN3O#~@qvQg=U^wotG%A1Pdfti`wY^Ch9f2+z|^=@_*A zYY|v3u2EN&Z|{x*)-2nfQ$Iolq9Nl+V{!!s6Lv9avnV-%Zt+(Lg1~g4v8=t{#gByy z4Us)?TFho#7IS?ZgyVq|GLX*C*rg?gZ=hB%IuVIH3mKC8u+2ZreMBkt z92_SuRR01SbjDf~Nr3!uHC_Sb$Rz1mEvhQ5 z6og+M)l7>4`UY%3UZIo!ebRvf~ z(dP&`1e|=FfCX7{AUFHY8urE=1g(mR<>{ka>Yn+J?5QC$pq$`-rfE~<2}BK+$fzWe z)?m(F@A^)5h3>_NQeTatjCa?_?1M#y;mA#v#K81mpWJ=Q!Jv(GD@2UtxHi*t@EHrW z53=0UbKU`IHof}C^_X25;0^jX6@)^bSRoA79^zh^f-uDurRRY(JXp7~+S-#MV(~)3F8dm2CA!nd>H4cz23~_*U{Pyo!iq$XO z;^sfh3z$ggsB16Lk?&Y?*H%KR6&YW=$kI6c@zNL5AJ?kpK0vXP zk;n7x=&X@ebBmHndeZ}yGRpdwv+>D}=qOCpt0xU?$6ZZnh`>8vG>&Q{O=))wLKK)U zUsBz@N%Mn6!q;ihCN+`IjY4v`eUv{JSHgHp3;OQEI!0jdy28yE=-k(w=GoF&QXK@< zqLn5D^DuNE3B47E4FUA$aJ6!0mp?O#}V# zwAjw&qOl=Zw;JYT!bn|QMpEsZzuLtCi1RJsCMVDmEgiM3tua>HX<%rcr60cPjuljY z-kNP;tDf%)GA0^=uvZ`QIn$V7gsJ9+Xdu#PrKUH%=MeM!9 zhKuidvT%J>hwUP8-MZe7U<7a%u&lnGZFs|bhYhQ~y?@qtxeP@RBvIr<*S~zbyKETj zD)syM&5v-{(@;A|LqSR07|g`7s#24WZhHjG?-MRu5of$~l(bdiVvUiZX;9mg9af~1 zg_zlc$x8|d4u=46%0~5l0h70d_TV|&yB%%z9c=}Ub7x(@ZSh4mzmmI2Gu@w>KobcoZbE z@)x;jAqo&9-|y3eJBjFLmhp88)K!dg?5E~;%|A59nwT|g{_q+zFZ724OQD)i4$E3I z?xCy&J~`t#4V<&5(@m*1VM|tk=V^=i*SNIz(ZY7|9o@4m^FQcQ*pUU$I?w2(TSmHy zTzP!o`j)4$!-P{nk^HSzsAr{9+tCzH=wAv!@{UC_e~Ls|*nESQw;okgPOTA0hN_n6 zs7mkhT+`3|H0tNlIS zTN>9P`q@ykJ}dlj2Fo+u8BsqZB_KiMtzgo4={yLmU`56XT&0!%`uR+vWhw3va|?i| zsyaoID^Q7tJ0JZk<9j(EAd6?IVKdTgr*pFxo=E%ld(|UndT8%b@c$M=sn(zEce;nP z20`gq>;}_cr+1B9`l}(zdj!OZ5=&GI_={*>J#DfM00|KCNfKNv&5Lu@t5a+U-peXL zbm54~K&O?z>+B(dEG(+9{37a|$&4tb>WGHs;p!TLG*-v={gc7})mI6l4aH;ND$GGI m-@?G&A^)`!-grSG0a%U`aZ$dN(L&U;0pz5VB+Dg?gZ~dVW!C%v literal 0 HcmV?d00001 diff --git a/website/static/img/institutions/shields-rounded-corners/edugain-shield-rounded-corners.png b/website/static/img/institutions/shields-rounded-corners/edugain-shield-rounded-corners.png new file mode 100644 index 0000000000000000000000000000000000000000..a42be72e9026427e92dfa49bab1ac2f3c210815d GIT binary patch literal 4454 zcmV-s5t;6ZP)KRsj*C zvPcvq0!G}>1jRk)p9* zL=k<7vdCx%t4owckfp1)+TN{hM7n9JtGcV22Kt^po~Et!*Z=z8@B8Z3ZJ4HM$lJtk z5-$QT=mFIbF9OiW#!9NOl5D61Z|Jsqs=dBbhQ6V&UnnwiA`(4Q8WQcwF5toJ(M080 zyy^$=FKyI1zV>}X6^q_NMkIJabLM-q7%$H!s(zr27~gf#iOUqT*LbbNK?8h^=l66u zgA>hk`5=(La$%Ymw4Dafm@XYKT|Uq$Vc4~A>uSC%dL}Y@rq?>G5nt|xSfB>W)tZ;T zg5?^oqB}2hd$0&Tpw z*K~28ZD4?^CBaP6EPW^*GHS`DD!Tm+)zRRsbvP^J0<)0ePPjl~(&6#a;R$vxYT&BC zTTG>23A1<01vKD61760nxt7okiAa*gEJPw_IILI5AA4OkZlU?>sdhQY(++UZAPq4T zWD>Q7K*F;4WwNmgxOYkn?x4}C#k+Ke*Sg4B8WcT{L4ykxFalYbV$PdL#0*~PS+~?q zqnxt^`|@R%4rKsmhQKaNKkfr6mM5-qaZ^Co;8MUVzI7oJABdeHunS{hE>NQS`7fMP zanFWMUWNbitqayMdZ2-(I_mMVuh7I*5}!~3Hw(5V9TG(~S3A8rkA;OgXx-5);<{*w zuSqLUdQDi$ovM5&0g}MZy(D}TEf{F3wO;#as{TZ-7@Q?TMvL?fI{oEIhsE*Rkx)kr zD>UXL?NGc3QTpHxhm3b_w9&lONLU+WMt z>LBV$TX(EbFWroeOL9;ZDk(A+7a1>KpfDXxnSIE8l%1*eNyC#otEdE|t^Mmeis?&~ zDKEfETWOOm4f@kZ0AliP6GVglwDY84l^TtC#7*HjpL)duYMut#(hP-P8_SD)UU(M? zi(<}AuQgsN(tm%1xK#t-(iChXD#b+i$(d@Y=G#v(HNsII37R>ddkt+nf3HYdLblYK z%8F4*EmD+8f$@(&WzM&Z^4UO|4(A2$pSxl1u8=oN& z6WFj$m?jMj=~IK|mv*BGFF1L{i**jI^_Fze_mvL}g@>@C|6^ryfg#GrmY@@+NtBwV z^~<`uBs$i8zCkvW^73aUd>%R7vp`#kG-)Xk5s!VjgJ^D4OqwT8oQn*P>I_IOoU%Ha z5fRd$q3#2H<2*lDF-u@n951$DQ#8*-*wy&a#-r&Jz$Q zG0_??I&*8;RB3gp@&xx8G|&R$X%`RI_;bA1G(7o$QHaE8Z=@0G!jaMAzo(sAXY`6s4^wR-s?@Lsh z?UNwLs5swsU)bd0EKRfYRavxE)NHCK*B<#ybU8Pc(Yi*C1%6l)f{b~}^_}tqO&hhk zueTV!`;#5JcxN~K~D&Yp84VNx>j&I0fpLbo({6Le9z%E`Sdf0`< zF@V5Lr6+)K^D7-F)dB0XZ9Z)K^G^JFjac0v)%&68E)1u0 zn4Mb|r+}@qMDUfaAi-`FWCnNI~|?L(xd|OrTs2{baS_%>MS>zioZ*NxZg{Q;t4;j$&Gh zRgin5w(X-2g_kpqk-rvkl(THiQ$kVZ5l5qwo(3;7f?}fbEP1_zY^-t=|6!)fCu9*; zMMb>2Oqc%`vZbNx_7!u7g}Q&CbeG;w7qR0<)lWPrWU%qeVVc0W6p~<7h%ELwA&U^a zR~VT9UW`e!)f3l?0dZ5sIl$Du;an|LFYSh2ukHAJ2i4rjxDAKXZQuWiEIbk&A1joc ziC>OzTv(p)I}kn#2wIp2IzbkW|9lRwI7i;8X0{1@)?(ijqI>n=F<_mA~I9A;EZ z-&b*WMR)UPh&2C6!l1Vd-myy>65{$nntuF%sN>|$(y*!Wq;0~wmN12X&;cqQU*+@L zPy99?lZ~;uafx3oyl)z%`m=|5%TT9l$;zNGS2IQ?*tlN6DR4&B=;RH8cMk%WNkJTq z#?LohE3|SCvuS7#{E=h8CmRvT3_Y-j-F#UXE?!X0PI1Ydul1FRW>Z0-A~N z$IgHiYI9lcewTQ)5cGl6y#_A6EDZ|rU-!1KfgRaY#>knQX-LCFscngJ&k(Tg{-?g+ zQ^Pr~ra|1SZOPa|30hskuYJZPpo3LI(McO1ahfQgm+OP?iHfum_x^aQH^}+ zR!HO~7HDbc=$`HYDBFxFMIMR_SfA~H_0}|FdAT!kF6U$#=)`p(;qI@X&F*29xuD~x zqp@~IpD?U9WU;WO0PDBk!V13QN4erNy1mx%K)SKZrNi8D>s>_9-Q9zUG@jnrYv(y4 zGGHScYtZiP%;CFc$ed-SveG){5IsSChpv%mU0q0ZaVM{$H})!9k@Bm0qYNPu#=}9na1Z- zN3ElRh6sdSe^%E-gZr6W=x_cqa|HI_y4 z9=6S}-kPa=jEx0$;&jVRy2Sm%G= z{D%juiQ1!N!$k+1^^HK|*7nUAK_1-vSFckhv$8(^z5WaXmuj!YukB%vnL(b7PJOSh z&c)NGk?>!=7NQ{CclIL<^%Jv{$x|3>zMR2c>xK~(`lwsOjQ*tPPS&rl^?BwQJMKR^ zJYd~Rj5S}$f%Hvm#tbM=c@I{G^x;|bk2(H*o>?BS&T4vWiCPtuxkcimCMv(6*sa-2 z8Mt%boPJ0?2}16@-#P3rC?gFS76N)dqWVkr2{L8lUhZ?sQ2&~|ieXK|_qOK* zZdxNxhz02%UY^5l&9d0nrNd`?!ol~yiMsLXT3J*CO<<;?RCCnKN;zz{ha7hQ`yCcQ z8Dd7myxor|L;dszZ-8Ua9d!;V^BqI%1#L~aVh?%e3`=5Sp^RV2@7TS}RF>}{huzO+ zawKn`CU5?Or2~|Wd(A>we>@ZPKDvKof&LSXTi{hRKg#&kT-Hd=Ot~CT&glA3cPz_P zdcvz{u8j%iuH`z=@-RU88~rz#HvgqN3Um#j@e90)c1+oHc`Hlq*Nt+9-9>bN&(@#W z?^U!z%G)49Z-0y=$gn4-OMJ{R-%kfvwqrQ6pK7V+@}%itR5YcnH5G4zNS*z1kxYd? z+DRES)$&~F2)3sjO(oxAhqAq(Z5yqv#f!JWq|R<2C-DiFPkvvbc~Hk-COuxBD~nyo z#=7_A?9`W)i_$>XhhOP6(lA~=NmH<9*&AL(JD@BL zO-2*nw@bN`^%H5bN7!NvU=|!=BHyw7zvH^TfKvQ z6x9G!G()-6zLsI1a5O%h7iEZz(4V z_6UzAs`rw=9`7Z$-YoGSFUVf-pU+`mXVW@ecaQFTn$i3Ql4}A|vyl;z15wcgMpWe5 zr7ZCqC!hS0P?RC_LwTx&tJ!)8J$CPpZxT0Z1{`PtGnKtdHvZGjG)7S`%Hq;pu?u_d zRHe($bI=>>v6k&yOg~>4IMzX9b*8d+dUk!VI$SnpHTYNPKA@^&Un(;TnRR&A-8w99 zz=t#i`TZtUIaA0Hr+t%xlMQEx+I%wkp``?`) z`eq-|>gk|~U-B5cuo1F67*?Wb$Wh8n&zpSOLy3&>oVoYMb5(}pJe z+IAYdZX3XjT*e4lls#&-e*u)Mo% z?O0&yT8{<|q6;8oE4z@Yq+w5p}rHy_PV)UCj*8rr^L0YdjP*apNF)(=x9# zd>`>5By5r3afMAYp4-Dx_;-?Z!D)zu0yjCyTC6ylYP(8Q?WT=Q-Bi2?874a7ZU<0L zHdGSVONhD>vc9^vGHGvdG(?!+6XxX!hyriGpE#tPJS8w~jYOmN0MNjZ>-KjAaKMWJ zVrlT>ZbNLhR|L56x>BO1guGqj3TVg#$-*N-^8Q=mqwY)V=aAGWB3LpN_@xlp@bb& zmm=a6GnR6kwXA1TFPg=dvS6SAW#$CNY!tjkAr@;#kD02tt@$LEO@zUR?r}BHgi1ru3RtD4a30>THXz`&-3f$ zaU%ZW4lKRsj*C zvPcvq0!G}>1jRk)p9* zL=k<7vdCx%t4owckfp1)+TN{hM7n9JtGcV22Kt^po~Et!*Z=z8@B8Z3ZJ4HM$lJtk z5-$QT=mFIbF9OiW#!9NOl5D61Z|Jsqs=dBbhQ6V&UnnwiA`(4Q8WQcwF5toJ(M080 zyy^$=FKyI1zV>}X6^q_NMkIJabLM-q7%$H!s(zr27~gf#iOUqT*LbbNK?8h^=l66u zgA>hk`5=(La$%Ymw4Dafm@XYKT|Uq$Vc4~A>uSC%dL}Y@rq?>G5nt|xSfB>W)tZ;T zg5?^oqB}2hd$0&Tpw z*K~28ZD4?^CBaP6EPW^*GHS`DD!Tm+)zRRsbvP^J0<)0ePPjl~(&6#a;R$vxYT&BC zTTG>23A1<01vKD61760nxt7okiAa*gEJPw_IILI5AA4OkZlU?>sdhQY(++UZAPq4T zWD>Q7K*F;4WwNmgxOYkn?x4}C#k+Ke*Sg4B8WcT{L4ykxFalYbV$PdL#0*~PS+~?q zqnxt^`|@R%4rKsmhQKaNKkfr6mM5-qaZ^Co;8MUVzI7oJABdeHunS{hE>NQS`7fMP zanFWMUWNbitqayMdZ2-(I_mMVuh7I*5}!~3Hw(5V9TG(~S3A8rkA;OgXx-5);<{*w zuSqLUdQDi$ovM5&0g}MZy(D}TEf{F3wO;#as{TZ-7@Q?TMvL?fI{oEIhsE*Rkx)kr zD>UXL?NGc3QTpHxhm3b_w9&lONLU+WMt z>LBV$TX(EbFWroeOL9;ZDk(A+7a1>KpfDXxnSIE8l%1*eNyC#otEdE|t^Mmeis?&~ zDKEfETWOOm4f@kZ0AliP6GVglwDY84l^TtC#7*HjpL)duYMut#(hP-P8_SD)UU(M? zi(<}AuQgsN(tm%1xK#t-(iChXD#b+i$(d@Y=G#v(HNsII37R>ddkt+nf3HYdLblYK z%8F4*EmD+8f$@(&WzM&Z^4UO|4(A2$pSxl1u8=oN& z6WFj$m?jMj=~IK|mv*BGFF1L{i**jI^_Fze_mvL}g@>@C|6^ryfg#GrmY@@+NtBwV z^~<`uBs$i8zCkvW^73aUd>%R7vp`#kG-)Xk5s!VjgJ^D4OqwT8oQn*P>I_IOoU%Ha z5fRd$q3#2H<2*lDF-u@n951$DQ#8*-*wy&a#-r&Jz$Q zG0_??I&*8;RB3gp@&xx8G|&R$X%`RI_;bA1G(7o$QHaE8Z=@0G!jaMAzo(sAXY`6s4^wR-s?@Lsh z?UNwLs5swsU)bd0EKRfYRavxE)NHCK*B<#ybU8Pc(Yi*C1%6l)f{b~}^_}tqO&hhk zueTV!`;#5JcxN~K~D&Yp84VNx>j&I0fpLbo({6Le9z%E`Sdf0`< zF@V5Lr6+)K^D7-F)dB0XZ9Z)K^G^JFjac0v)%&68E)1u0 zn4Mb|r+}@qMDUfaAi-`FWCnNI~|?L(xd|OrTs2{baS_%>MS>zioZ*NxZg{Q;t4;j$&Gh zRgin5w(X-2g_kpqk-rvkl(THiQ$kVZ5l5qwo(3;7f?}fbEP1_zY^-t=|6!)fCu9*; zMMb>2Oqc%`vZbNx_7!u7g}Q&CbeG;w7qR0<)lWPrWU%qeVVc0W6p~<7h%ELwA&U^a zR~VT9UW`e!)f3l?0dZ5sIl$Du;an|LFYSh2ukHAJ2i4rjxDAKXZQuWiEIbk&A1joc ziC>OzTv(p)I}kn#2wIp2IzbkW|9lRwI7i;8X0{1@)?(ijqI>n=F<_mA~I9A;EZ z-&b*WMR)UPh&2C6!l1Vd-myy>65{$nntuF%sN>|$(y*!Wq;0~wmN12X&;cqQU*+@L zPy99?lZ~;uafx3oyl)z%`m=|5%TT9l$;zNGS2IQ?*tlN6DR4&B=;RH8cMk%WNkJTq z#?LohE3|SCvuS7#{E=h8CmRvT3_Y-j-F#UXE?!X0PI1Ydul1FRW>Z0-A~N z$IgHiYI9lcewTQ)5cGl6y#_A6EDZ|rU-!1KfgRaY#>knQX-LCFscngJ&k(Tg{-?g+ zQ^Pr~ra|1SZOPa|30hskuYJZPpo3LI(McO1ahfQgm+OP?iHfum_x^aQH^}+ zR!HO~7HDbc=$`HYDBFxFMIMR_SfA~H_0}|FdAT!kF6U$#=)`p(;qI@X&F*29xuD~x zqp@~IpD?U9WU;WO0PDBk!V13QN4erNy1mx%K)SKZrNi8D>s>_9-Q9zUG@jnrYv(y4 zGGHScYtZiP%;CFc$ed-SveG){5IsSChpv%mU0q0ZaVM{$H})!9k@Bm0qYNPu#=}9na1Z- zN3ElRh6sdSe^%E-gZr6W=x_cqa|HI_y4 z9=6S}-kPa=jEx0$;&jVRy2Sm%G= z{D%juiQ1!N!$k+1^^HK|*7nUAK_1-vSFckhv$8(^z5WaXmuj!YukB%vnL(b7PJOSh z&c)NGk?>!=7NQ{CclIL<^%Jv{$x|3>zMR2c>xK~(`lwsOjQ*tPPS&rl^?BwQJMKR^ zJYd~Rj5S}$f%Hvm#tbM=c@I{G^x;|bk2(H*o>?BS&T4vWiCPtuxkcimCMv(6*sa-2 z8Mt%boPJ0?2}16@-#P3rC?gFS76N)dqWVkr2{L8lUhZ?sQ2&~|ieXK|_qOK* zZdxNxhz02%UY^5l&9d0nrNd`?!ol~yiMsLXT3J*CO<<;?RCCnKN;zz{ha7hQ`yCcQ z8Dd7myxor|L;dszZ-8Ua9d!;V^BqI%1#L~aVh?%e3`=5Sp^RV2@7TS}RF>}{huzO+ zawKn`CU5?Or2~|Wd(A>we>@ZPKDvKof&LSXTi{hRKg#&kT-Hd=Ot~CT&glA3cPz_P zdcvz{u8j%iuH`z=@-RU88~rz#HvgqN3Um#j@e90)c1+oHc`Hlq*Nt+9-9>bN&(@#W z?^U!z%G)49Z-0y=$gn4-OMJ{R-%kfvwqrQ6pK7V+@}%itR5YcnH5G4zNS*z1kxYd? z+DRES)$&~F2)3sjO(oxAhqAq(Z5yqv#f!JWq|R<2C-DiFPkvvbc~Hk-COuxBD~nyo z#=7_A?9`W)i_$>XhhOP6(lA~=NmH<9*&AL(JD@BL zO-2*nw@bN`^%H5bN7!NvU=|!=BHyw7zvH^TfKvQ z6x9G!G()-6zLsI1a5O%h7iEZz(4V z_6UzAs`rw=9`7Z$-YoGSFUVf-pU+`mXVW@ecaQFTn$i3Ql4}A|vyl;z15wcgMpWe5 zr7ZCqC!hS0P?RC_LwTx&tJ!)8J$CPpZxT0Z1{`PtGnKtdHvZGjG)7S`%Hq;pu?u_d zRHe($bI=>>v6k&yOg~>4IMzX9b*8d+dUk!VI$SnpHTYNPKA@^&Un(;TnRR&A-8w99 zz=t#i`TZtUIaA0Hr+t%xlMQEx+I%wkp``?`) z`eq-|>gk|~U-B5cuo1F67*?Wb$Wh8n&zpSOLy3&>oVoYMb5(}pJe z+IAYdZX3XjT*e4lls#&-e*u)Mo% z?O0&yT8{<|q6;8oE4z@Yq+w5p}rHy_PV)UCj*8rr^L0YdjP*apNF)(=x9# zd>`>5By5r3afMAYp4-Dx_;-?Z!D)zu0yjCyTC6ylYP(8Q?WT=Q-Bi2?874a7ZU<0L zHdGSVONhD>vc9^vGHGvdG(?!+6XxX!hyriGpE#tPJS8w~jYOmN0MNjZ>-KjAaKMWJ zVrlT>ZbNLhR|L56x>BO1guGqj3TVg#$-*N-^8Q=mqwY)V=aAGWB3LpN_@xlp@bb& zmm=a6GnR6kwXA1TFPg=dvS6SAW#$CNY!tjkAr@;#kD02tt@$LEO@zUR?r}BHgi1ru3RtD4a30>THXz`&-3f$ zaU%ZW4l Date: Tue, 20 Oct 2026 17:30:46 +0900 Subject: [PATCH 41/41] =?UTF-8?q?=E5=88=A5=E3=83=97=E3=83=AD=E3=82=B8?= =?UTF-8?q?=E3=82=A7=E3=82=AF=E3=83=88=E3=81=AEWiki=E3=82=92=E8=A6=AA?= =?UTF-8?q?=E3=81=AB=E8=A8=AD=E5=AE=9A=E3=81=A7=E3=81=8D=E3=81=AA=E3=81=84?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addons/wiki/models.py | 11 +++++- addons/wiki/tests/test_wiki.py | 63 ++++++++++++++++++++++++++++++++++ addons/wiki/views.py | 9 ++++- 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/addons/wiki/models.py b/addons/wiki/models.py index e20f40dd730..f8c6a1bf75f 100644 --- a/addons/wiki/models.py +++ b/addons/wiki/models.py @@ -266,7 +266,16 @@ def get_for_node(self, node, name=None, id=None): deleted__isnull=True, node=node, ).order_by('created', 'id').first() - return WikiPage.load(id) + if id: + wiki_page = WikiPage.load(id) + # Restrict by node so wid cannot leak another project's page name. + if ( + wiki_page + and wiki_page.deleted is None + and wiki_page.node_id == node.id + ): + return wiki_page + return None def get_for_child_nodes(self, node, parent=None): if parent: diff --git a/addons/wiki/tests/test_wiki.py b/addons/wiki/tests/test_wiki.py index 6daffd6dead..bd713a52977 100644 --- a/addons/wiki/tests/test_wiki.py +++ b/addons/wiki/tests/test_wiki.py @@ -448,6 +448,17 @@ def test_wiki_id_url_get_returns_404(self): res = self.app.get(url, expect_errors=True) assert_equal(res.status_code, 404) + def test_wiki_id_url_get_returns_404_for_other_node_wiki(self): + other = ProjectFactory(is_public=True, creator=self.user) + other_page = WikiPage.objects.create_for_node( + other, 'SecretFromOther', 'content', Auth(self.user) + ) + url = self.project.web_url_for( + 'project_wiki_id_page', wid=other_page._primary_key, _guid=True + ) + res = self.app.get(url, expect_errors=True) + assert_equal(res.status_code, 404) + def test_home_is_capitalized_in_web_view(self): url = self.project.web_url_for('project_wiki_home', wid='home', _guid=True) res = self.app.get(url, auth=self.user.auth).follow(auth=self.user.auth) @@ -3340,6 +3351,24 @@ def test_bulk_update_wiki_sort(self): self.assertEqual(result_wiki_child_page2, {'parent_id': wiki_page2_id, 'sort_order': 1}) self.assertEqual(result_wiki_child_page3, {'parent_id': wiki_child_page2_id, 'sort_order': 1}) + def test_bulk_update_wiki_sort_rejects_cross_node_parent(self): + other = ProjectFactory(is_public=True, creator=self.user) + other_page = WikiPage.objects.create_for_node( + other, 'other project wiki', 'content', self.consolidate_auth + ) + other_guid = other_page.guids.first()._id + parent_before = self.wiki_page1.parent_id + sort_id_list = [self.guid1] + sort_num_list = [1] + parent_wiki_id_list = [other_guid] + with self.assertRaises(HTTPError) as cm: + views._bulk_update_wiki_sort( + self.project, sort_id_list, sort_num_list, parent_wiki_id_list + ) + self.assertEqual(cm.exception.code, http_status.HTTP_400_BAD_REQUEST) + self.wiki_page1.reload() + self.assertEqual(self.wiki_page1.parent_id, parent_before) + def _wiki_sort_payload(self): return { 'sortedData': [ @@ -3380,3 +3409,37 @@ def test_project_update_wiki_page_sort(self): self.assertEqual(result_wiki_child_page1, {'parent_id': wiki_page2_id, 'sort_order': 1}) self.assertEqual(result_wiki_child_page2, {'parent_id': wiki_page2_id, 'sort_order': 2}) self.assertEqual(result_wiki_child_page3, {'parent_id': wiki_child_page2_id, 'sort_order': 1}) + + def test_project_update_wiki_page_sort_rejects_cross_node_parent(self): + other = ProjectFactory(is_public=True, creator=self.user) + other_page = WikiPage.objects.create_for_node( + other, 'other project wiki', 'content', self.consolidate_auth + ) + other_guid = other_page.guids.first()._id + parent_before = self.wiki_page1.parent_id + url = self.project.api_url_for('project_update_wiki_page_sort') + # Nest a local wiki under another project's wiki id as parent. + payload = { + 'sortedData': [ + { + 'name': 'other as parent', + 'id': other_guid, + 'sortOrder': 1, + 'children': [ + { + 'name': 'wiki page1', + 'id': self.guid1, + 'sortOrder': 1, + 'children': [], + 'fold': False, + } + ], + 'fold': False, + } + ] + } + res = self.app.post_json(url, payload, auth=self.user.auth, expect_errors=True) + assert_equal(res.status_code, http_status.HTTP_400_BAD_REQUEST) + self.wiki_page1.reload() + self.assertEqual(self.wiki_page1.parent_id, parent_before) + diff --git a/addons/wiki/views.py b/addons/wiki/views.py index a1051ba9110..f7c3602f4ad 100644 --- a/addons/wiki/views.py +++ b/addons/wiki/views.py @@ -93,6 +93,10 @@ message_short='Invalid request', message_long='The requested version of this wiki page does not exist.' )) +WIKI_INVALID_PARENT_ERROR = HTTPError(http_status.HTTP_400_BAD_REQUEST, data=dict( + message_short='Invalid parent wiki', + message_long='Parent wiki page does not exist on this project.', +)) WIKI_IMPORT_TASK_ALREADY_EXISTS = HTTPError(http_status.HTTP_400_BAD_REQUEST, data=dict( message_short='Running Task exists', @@ -1408,7 +1412,10 @@ def _bulk_update_wiki_sort(node, sort_id_list, sort_num_list, parent_wiki_id_lis parent_wiki_id = parent_wiki_id_list[idx] setattr(page, 'sort_order', sort_order_number) if parent_wiki_id is not None: - parent = WikiPage.objects.get(guids___id=parent_wiki_id) + # Same-node only; global guid lookup would allow cross-project parents. + parent = WikiPage.objects.get_for_node(node, id=parent_wiki_id) + if parent is None: + raise WIKI_INVALID_PARENT_ERROR setattr(page, 'parent', parent) else: setattr(page, 'parent', None)