diff --git a/Dockerfile b/Dockerfile index 4db505fddee..47b7d156f57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,6 @@ COPY ./addons/onedrive/requirements.txt /code/addons/onedrive/ COPY ./addons/onedrivebusiness/requirements.txt /code/addons/onedrivebusiness/ #COPY ./addons/osfstorage/requirements.txt ./addons/osfstorage/ COPY ./addons/owncloud/requirements.txt ./addons/owncloud/ -COPY ./addons/s3/requirements.txt ./addons/s3/ COPY ./addons/twofactor/requirements.txt ./addons/twofactor/ #COPY ./addons/wiki/requirements.txt ./addons/wiki/ COPY ./addons/zotero/requirements.txt ./addons/zotero/ diff --git a/addons/base/views.py b/addons/base/views.py index c1bb834b7b0..d373dc6815f 100644 --- a/addons/base/views.py +++ b/addons/base/views.py @@ -66,6 +66,8 @@ from website.ember_osf_web.decorators import ember_flag_is_active from website.project.utils import serialize_node from website.util import rubeus, timestamp, inspect_info # noqa +import re +import json as _json from osf.features import ( SLOAN_COI_DISPLAY, @@ -433,6 +435,14 @@ def get_auth(auth, **kwargs): credentials = node.serialize_waterbutler_credentials(provider_name) waterbutler_settings = node.serialize_waterbutler_settings(provider_name) + if is_node_process and provider_settings: + # Same limit the file browser shows; + # Only osfstorage defines `high_max_file_size`. + max_size = getattr(provider_settings.config, 'max_file_size', None) + if auth.user and 'high_upload_limit' in auth.user.system_tags: + max_size = getattr(provider_settings.config, 'high_max_file_size', None) or max_size + waterbutler_settings['max_file_size'] = max_size + if not is_node_process: # for only location_id value storage = ExportDataLocation.objects.get(pk=location_id) @@ -582,6 +592,33 @@ def create_waterbutler_log(payload, **kwargs): params=payload ) if payload.get('email') is True or payload.get('errors'): + # Parse structured error info from WaterButler exception repr strings + # e.g. "" + error_info = None + if payload.get('errors'): + for err_str in payload['errors']: + # Try to extract JSON payload embedded in the repr string + json_match = re.search(r'\((?:\d+),\s*(\{.*\})\)', err_str, re.DOTALL) + if json_match: + try: + err_data = _json.loads(json_match.group(1)) + message_key = err_data.get('message_key', '') + if message_key == 'quota_exceeded' or 'quota_exceeded' in err_str: + error_info = {'type': 'quota_exceeded'} + break + elif err_data.get('oversized_files'): + error_info = { + 'type': 'oversized', + 'oversized_files': err_data['oversized_files'], + 'max_size': err_data.get('max_size'), + } + break + except Exception as e: + logger.warning( + f'Failed to parse WaterButler error representation string: {err_str}. ' + f'Error: {e}' + ) + mails.send_mail( user.username, mails.FILE_OPERATION_FAILED if payload.get('errors') @@ -592,7 +629,8 @@ def create_waterbutler_log(payload, **kwargs): source_path=payload['source']['materialized'], source_addon=payload['source']['addon'], destination_addon=payload['destination']['addon'], - osf_support_email=settings.OSF_SUPPORT_EMAIL + osf_support_email=settings.OSF_SUPPORT_EMAIL, + error_info=error_info, ) if payload.get('errors'): # Action failed but our function succeeded diff --git a/addons/binderhub/settings/defaults.py b/addons/binderhub/settings/defaults.py index 0a59f0fe19c..b22b155c219 100644 --- a/addons/binderhub/settings/defaults.py +++ b/addons/binderhub/settings/defaults.py @@ -44,6 +44,7 @@ ] MATLAB_RELEASES = [ + 'R2026b', 'R2026a', 'R2025b', 'R2025a', @@ -56,6 +57,7 @@ ] MATLAB_PRODUCTNAMES_MAP = { + 'R2026b': gen_product_name_list('addons/binderhub/settings/mpm_inputs/mpm_input_r2026b.txt'), 'R2026a': gen_product_name_list('addons/binderhub/settings/mpm_inputs/mpm_input_r2026a.txt'), 'R2025b': gen_product_name_list('addons/binderhub/settings/mpm_inputs/mpm_input_r2025b.txt'), 'R2025a': gen_product_name_list('addons/binderhub/settings/mpm_inputs/mpm_input_r2025a.txt'), diff --git a/addons/binderhub/settings/mpm_inputs/mpm_input_r2025a.txt b/addons/binderhub/settings/mpm_inputs/mpm_input_r2025a.txt index 3e012822ef1..7570f6f2b3e 100644 --- a/addons/binderhub/settings/mpm_inputs/mpm_input_r2025a.txt +++ b/addons/binderhub/settings/mpm_inputs/mpm_input_r2025a.txt @@ -71,6 +71,24 @@ # destinationFolder= +######################################################################## +## SUPPORT PACKAGE DESTINATION +######################################################################## +## +## Specify the full path to the folder where you want to install +## support packages. If not specified, support packages are installed +## to the default support package location. +## +## Example: +## (Windows) supportPackageDestinationFolder=C:\shared\spkgs +## (Linux) supportPackageDestinationFolder=/home//spkgs +## (Mac) supportPackageDestinationFolder=/home//spkgs +## +## Set the desired value for supportPackageDestinationFolder and +## uncomment the following line. + +# supportPackageDestinationFolder= + ######################################################################## ## ACCEPT VENDOR LICENSE AGREEMENTS ######################################################################## diff --git a/addons/binderhub/settings/mpm_inputs/mpm_input_r2025b.txt b/addons/binderhub/settings/mpm_inputs/mpm_input_r2025b.txt index 3ae7663149d..2340f6e82da 100644 --- a/addons/binderhub/settings/mpm_inputs/mpm_input_r2025b.txt +++ b/addons/binderhub/settings/mpm_inputs/mpm_input_r2025b.txt @@ -71,6 +71,24 @@ # destinationFolder= +######################################################################## +## SUPPORT PACKAGE DESTINATION +######################################################################## +## +## Specify the full path to the folder where you want to install +## support packages. If not specified, support packages are installed +## to the default support package location. +## +## Example: +## (Windows) supportPackageDestinationFolder=C:\shared\spkgs +## (Linux) supportPackageDestinationFolder=/home//spkgs +## (Mac) supportPackageDestinationFolder=/home//spkgs +## +## Set the desired value for supportPackageDestinationFolder and +## uncomment the following line. + +# supportPackageDestinationFolder= + ######################################################################## ## ACCEPT VENDOR LICENSE AGREEMENTS ######################################################################## diff --git a/addons/binderhub/settings/mpm_inputs/mpm_input_r2026a.txt b/addons/binderhub/settings/mpm_inputs/mpm_input_r2026a.txt index 96014b83127..7c8376a7f47 100644 --- a/addons/binderhub/settings/mpm_inputs/mpm_input_r2026a.txt +++ b/addons/binderhub/settings/mpm_inputs/mpm_input_r2026a.txt @@ -71,6 +71,24 @@ # destinationFolder= +######################################################################## +## SUPPORT PACKAGE DESTINATION +######################################################################## +## +## Specify the full path to the folder where you want to install +## support packages. If not specified, support packages are installed +## to the default support package location. +## +## Example: +## (Windows) supportPackageDestinationFolder=C:\shared\spkgs +## (Linux) supportPackageDestinationFolder=/home//spkgs +## (Mac) supportPackageDestinationFolder=/home//spkgs +## +## Set the desired value for supportPackageDestinationFolder and +## uncomment the following line. + +# supportPackageDestinationFolder= + ######################################################################## ## ACCEPT VENDOR LICENSE AGREEMENTS ######################################################################## diff --git a/addons/binderhub/settings/mpm_inputs/mpm_input_r2026b.txt b/addons/binderhub/settings/mpm_inputs/mpm_input_r2026b.txt new file mode 100644 index 00000000000..f6fc9260862 --- /dev/null +++ b/addons/binderhub/settings/mpm_inputs/mpm_input_r2026b.txt @@ -0,0 +1,514 @@ +######################################################################## +## Configuration File for Installing or Downloading R2026b MathWorks Products +######################################################################## +## +## Use this file to configure the installation or download of MathWorks +## products and support packages from the command line using the +## MATLAB Package Manager (mpm). +## +## To configure your MATLAB installation or download: +## +## 1. Set configuration parameters by uncommenting lines that +## start with a single '#' and updating the values. The +## comments above each parameter describe the valid values. +## +## 2. Run mpm from the command line, using the --inputfile option +## to specify the full path to this configuration file. +## +## Install products: +## mpm install --inputfile +## +## Download products without installing: +## mpm download --inputfile +## +## You can download template input files for all supported releases from +## https://www.mathworks.com/mpm_input_files +## +######################################################################## + + +######################################################################## +## RELEASE +######################################################################## +## +## By default, mpm gets the latest versions of R2026b MathWorks products +## and support packages using this input file. +## +## To get a specific update for R2026b, set the desired update +## number (for example, 1, 2, 3, and so on) and uncomment the following +## line. An updateLevel of 0 gets the general release. + +# updateLevel=0 + + +######################################################################## +## INSTALLATION OR DOWNLOAD FOLDER +######################################################################## +## +## When installing products using 'mpm install': +## +## Specify the full path to the folder where you want to install the +## products. When adding products or support packages to an +## existing MATLAB installation, specify the full path to the folder +## where MATLAB is installed. +## +## Example: +## (Windows) destinationFolder=C:\Program Files\MATLAB\RXXXX +## (Linux) destinationFolder=/usr/local/RXXXX +## (macOS) destinationFolder=/Applications/MATLAB/RXXXX +## +## When downloading products using 'mpm download': +## +## Specify the full path to the folder where you want to download the +## products. For 'mpm download', this option is required. +## +## Example: +## (Windows) destinationFolder=C:\Users\\Downloads\MathWorks +## (Linux/macOS) destinationFolder=/home//downloads/mathworks +## +## Set the desired value for destinationFolder and +## uncomment the following line. + +# destinationFolder= + +######################################################################## +## SUPPORT PACKAGE DESTINATION +######################################################################## +## +## Specify the full path to the folder where you want to install +## support packages. If not specified, support packages are installed +## to the default support package location. +## +## Example: +## (Windows) supportPackageDestinationFolder=C:\shared\spkgs +## (Linux) supportPackageDestinationFolder=/home//spkgs +## (Mac) supportPackageDestinationFolder=/home//spkgs +## +## Set the desired value for supportPackageDestinationFolder and +## uncomment the following line. + +# supportPackageDestinationFolder= + +######################################################################## +## ACCEPT VENDOR LICENSE AGREEMENTS +######################################################################## +## +## To install support packages that contain vendor licenses, you must agree +## to accept these licenses. +## +## Uncomment the following line to accept all vendor license agreements. +## When you run mpm with this option, mpm displays the accepted agreements. + +# acceptVendorLicenses=yes + +######################################################################## +## PRODUCTS +######################################################################## +## +## Uncomment the lines for the products you want to install or download. + +#product.5G_Toolbox +#product.AUTOSAR_Blockset +#product.Aerospace_Blockset +#product.Aerospace_Toolbox +#product.Antenna_Toolbox +#product.Audio_Toolbox +#product.Automated_Driving_Toolbox +#product.Bioinformatics_Toolbox +#product.Bluetooth_Toolbox +#product.C2000_Microcontroller_Blockset +#product.Communications_Toolbox +#product.Computer_Vision_Toolbox +#product.Control_System_Toolbox +#product.Curve_Fitting_Toolbox +#product.DDS_Blockset +#product.DSP_HDL_Toolbox +#product.DSP_System_Toolbox +#product.Data_Acquisition_Toolbox +#product.Database_Toolbox +#product.Datafeed_Toolbox +#product.Deep_Learning_HDL_Toolbox +#product.Deep_Learning_Toolbox +#product.Econometrics_Toolbox +#product.Embedded_Coder +#product.Financial_Instruments_Toolbox +#product.Financial_Toolbox +#product.Fixed-Point_Designer +#product.Fuzzy_Logic_Toolbox +#product.GPU_Coder +#product.Global_Optimization_Toolbox +#product.HDL_Coder +#product.HDL_Verifier +#product.Image_Acquisition_Toolbox +#product.Image_Processing_Toolbox +#product.Industrial_Communication_Toolbox +#product.Instrument_Control_Toolbox +#product.LTE_Toolbox +#product.MATLAB +#product.MATLAB_Coder +#product.MATLAB_Compiler +#product.MATLAB_Compiler_SDK +#product.MATLAB_Copilot +#product.MATLAB_Parallel_Server +#product.MATLAB_Production_Server +#product.MATLAB_Report_Generator +#product.MATLAB_Test +#product.MATLAB_Web_App_Server +#product.Mapping_Toolbox +#product.Medical_Imaging_Toolbox +#product.Mixed-Signal_Blockset +#product.Model_Predictive_Control_Toolbox +#product.Model-Based_Calibration_Toolbox +#product.Motor_Control_Blockset +#product.Navigation_Toolbox +#product.Optimization_Toolbox +#product.Parallel_Computing_Toolbox +#product.Partial_Differential_Equation_Toolbox +#product.Phased_Array_System_Toolbox +#product.Point_Cloud_Toolbox +#product.Polyspace_Bug_Finder +#product.Polyspace_Bug_Finder_Server +#product.Polyspace_Code_Prover +#product.Polyspace_Code_Prover_Server +#product.Polyspace_Test +#product.Polyspace_Test_Server +#product.Polyspace_as_You_Code +#product.Powertrain_Blockset +#product.Predictive_Maintenance_Toolbox +#product.RF_Blockset +#product.RF_PCB_Toolbox +#product.RF_Toolbox +#product.ROS_Toolbox +#product.Radar_Toolbox +#product.Raspberry_Pi_Blockset +#product.Reinforcement_Learning_Toolbox +#product.Requirements_Toolbox +#product.Risk_Management_Toolbox +#product.RoadRunner +#product.Robotics_System_Toolbox +#product.Robust_Control_Toolbox +#product.STM32_Microcontroller_Blockset +#product.Satellite_Communications_Toolbox +#product.Sensor_Fusion_and_Tracking_Toolbox +#product.SerDes_Toolbox +#product.Signal_Integrity_Toolbox +#product.Signal_Processing_Toolbox +#product.SimBiology +#product.SimEvents +#product.Simscape +#product.Simscape_Battery +#product.Simscape_Driveline +#product.Simscape_Electrical +#product.Simscape_Fluids +#product.Simscape_Multibody +#product.Simulink +#product.Simulink_3D_Animation +#product.Simulink_Check +#product.Simulink_Coder +#product.Simulink_Compiler +#product.Simulink_Control_Design +#product.Simulink_Copilot +#product.Simulink_Coverage +#product.Simulink_Design_Optimization +#product.Simulink_Design_Verifier +#product.Simulink_Desktop_Real-Time +#product.Simulink_FMU_Builder +#product.Simulink_Fault_Analyzer +#product.Simulink_PLC_Coder +#product.Simulink_Real-Time +#product.Simulink_Report_Generator +#product.Simulink_Test +#product.Simulink_Variant_Manager +#product.SoC_Blockset +#product.Spreadsheet_Link +#product.Stateflow +#product.Statistics_and_Machine_Learning_Toolbox +#product.Symbolic_Math_Toolbox +#product.System_Composer +#product.System_Identification_Toolbox +#product.Text_Analytics_Toolbox +#product.UAV_Toolbox +#product.Vehicle_Dynamics_Blockset +#product.Vehicle_Network_Toolbox +#product.Vision_HDL_Toolbox +#product.Visual_Inspection_Toolbox +#product.WLAN_Toolbox +#product.Wavelet_Toolbox +#product.Wireless_HDL_Toolbox +#product.Wireless_Network_Toolbox +#product.Wireless_Testbench + + +######################################################################## +## SUPPORT PACKAGES +######################################################################## +## +## Uncomment the lines for the support packages you want to install or download. + +#product.3D_Asset_Processing_Library_for_MATLAB +#product.6G_Exploration_Library_for_5G_Toolbox +#product.AI_Verification_Library_for_Deep_Learning_Toolbox +#product.ASIC_Testbench_for_HDL_Verifier +#product.Aerospace_Blockset_Interface_for_Unreal_Engine_Projects +#product.Aerospace_Blockset_Maritime_Applications_Library +#product.Audio_Toolbox_Interface_for_SpeechBrain_and_Torchaudio_Libraries +#product.Automated_Driving_Toolbox_Importer_for_Zenrin_Japan_Map_API_3.0_(Itsumo_NAVI_API_3.0)_Service +#product.Automated_Driving_Toolbox_Interface_for_Eclipse_SUMO_Traffic_Simulator +#product.Automated_Driving_Toolbox_Interface_for_Unreal_Engine_Projects +#product.Automated_Driving_Toolbox_Model_for_Lidar_Lane_Detection +#product.Automated_Driving_Toolbox_Test_Suite_for_Euro_NCAP_Protocols +#product.CI/CD_Automation_for_Simulink_Check +#product.Communications_Toolbox_Support_Package_for_Analog_Devices_ADALM-Pluto_Radio +#product.Communications_Toolbox_Support_Package_for_RTL-SDR_Radio +#product.Communications_Toolbox_Support_Package_for_USRP_Radio +#product.Component_Deployment_Guideline_for_Embedded_Coder +#product.Computer_Vision_Toolbox_Interface_for_Nerfstudio_Library +#product.Computer_Vision_Toolbox_Interface_for_OpenCV_in_MATLAB +#product.Computer_Vision_Toolbox_Interface_for_OpenCV_in_Simulink +#product.Computer_Vision_Toolbox_Model_for_Apple_DepthPro_Network +#product.Computer_Vision_Toolbox_Model_for_BiSeNet_v2_Semantic_Segmentation_Network +#product.Computer_Vision_Toolbox_Model_for_Grounding_DINO_Object_Detection +#product.Computer_Vision_Toolbox_Model_for_Inflated-3D_Video_Classification +#product.Computer_Vision_Toolbox_Model_for_MapAnything_Network +#product.Computer_Vision_Toolbox_Model_for_Mask_R-CNN_Instance_Segmentation +#product.Computer_Vision_Toolbox_Model_for_Object_Keypoint_Detection +#product.Computer_Vision_Toolbox_Model_for_OpenAI_CLIP_Network +#product.Computer_Vision_Toolbox_Model_for_Pose_Mask_R-CNN_6-DoF_Object_Pose_Estimation +#product.Computer_Vision_Toolbox_Model_for_R(2+1)D_Video_Classification +#product.Computer_Vision_Toolbox_Model_for_RAFT_Optical_Flow_Estimation +#product.Computer_Vision_Toolbox_Model_for_RTMDet_Object_Detection +#product.Computer_Vision_Toolbox_Model_for_RetinaFace_Face_Detection +#product.Computer_Vision_Toolbox_Model_for_SOLOv2_Instance_Segmentation +#product.Computer_Vision_Toolbox_Model_for_SlowFast_Video_Classification +#product.Computer_Vision_Toolbox_Model_for_Text_Detection +#product.Computer_Vision_Toolbox_Model_for_Vision_Transformer_Network +#product.Computer_Vision_Toolbox_Model_for_YOLO_v2_Object_Detection +#product.Computer_Vision_Toolbox_Model_for_YOLO_v3_Object_Detection +#product.Computer_Vision_Toolbox_Model_for_YOLO_v4_Object_Detection +#product.Computer_Vision_Toolbox_Model_for_moondream_Vision_Language_Model +#product.Computer_Vision_Toolbox_OCR_Language_Data +#product.Data_Acquisition_Toolbox_Support_Package_for_Analog_Devices_ADALM1000_Hardware +#product.Data_Acquisition_Toolbox_Support_Package_for_Digilent_Analog_Discovery_Hardware +#product.Data_Acquisition_Toolbox_Support_Package_for_Measurement_Computing_Hardware +#product.Data_Acquisition_Toolbox_Support_Package_for_National_Instruments_NI-DAQmx_Devices +#product.Data_Acquisition_Toolbox_Support_Package_for_Windows_Sound_Cards +#product.Database_Toolbox_Interface_for_Neo4j_Bolt_Protocol +#product.Databricks_ODBC_Driver_for_Database_Toolbox +#product.Deep_Learning_Toolbox_Converter_for_ONNX_Model_Format +#product.Deep_Learning_Toolbox_Converter_for_PyTorch_Model_Format +#product.Deep_Learning_Toolbox_Converter_for_TensorFlow_models +#product.Deep_Learning_Toolbox_Importer_for_Caffe_Models +#product.Deep_Learning_Toolbox_Interface_for_LiteRT_Library +#product.Deep_Learning_Toolbox_Interface_for_alpha-beta-CROWN_Verifier +#product.Deep_Learning_Toolbox_Model_Compression_Library +#product.Deep_Learning_Toolbox_Model_for_AlexNet_Network +#product.Deep_Learning_Toolbox_Model_for_DarkNet-19_Network +#product.Deep_Learning_Toolbox_Model_for_DarkNet-53_Network +#product.Deep_Learning_Toolbox_Model_for_DenseNet-201_Network +#product.Deep_Learning_Toolbox_Model_for_EfficientNet-b0_Network +#product.Deep_Learning_Toolbox_Model_for_GoogLeNet_Network +#product.Deep_Learning_Toolbox_Model_for_Inception-ResNet-v2_Network +#product.Deep_Learning_Toolbox_Model_for_Inception-v3_Network +#product.Deep_Learning_Toolbox_Model_for_MobileNet-v2_Network +#product.Deep_Learning_Toolbox_Model_for_NASNet-Large_Network +#product.Deep_Learning_Toolbox_Model_for_NASNet-Mobile_Network +#product.Deep_Learning_Toolbox_Model_for_Places365-GoogLeNet_Network +#product.Deep_Learning_Toolbox_Model_for_ResNet-101_Network +#product.Deep_Learning_Toolbox_Model_for_ResNet-18_Network +#product.Deep_Learning_Toolbox_Model_for_ResNet-50_Network +#product.Deep_Learning_Toolbox_Model_for_ShuffleNet_Network +#product.Deep_Learning_Toolbox_Model_for_VGG-16_Network +#product.Deep_Learning_Toolbox_Model_for_VGG-19_Network +#product.Deep_Learning_Toolbox_Model_for_Xception_Network +#product.Embedded_Coder_Interface_to_QEMU_Emulator +#product.Embedded_Coder_Support_Package_for_ARM_Cortex-A_Processors +#product.Embedded_Coder_Support_Package_for_ARM_Cortex-M_Processors +#product.Embedded_Coder_Support_Package_for_ARM_Cortex-R_Processors +#product.Embedded_Coder_Support_Package_for_BeagleBone_Black_Hardware +#product.Embedded_Coder_Support_Package_for_Infineon_AURIX_TC3x_Microcontrollers +#product.Embedded_Coder_Support_Package_for_Infineon_AURIX_TC4x_Microcontrollers +#product.Embedded_Coder_Support_Package_for_Qualcomm_Hexagon_Processors +#product.Embedded_Coder_Support_Package_for_Real-Time_Linux +#product.Embedded_Coder_Support_Package_for_Renesas_RA_Microcontrollers +#product.Embedded_Coder_Support_Package_for_Renesas_RH850_Microcontrollers +#product.Embedded_Coder_Support_Package_for_Service-Oriented_Applications_on_Linux +#product.Ephemeris_Data_for_Aerospace_Toolbox +#product.Extended_Tire_Features_for_Vehicle_Dynamics_Blockset +#product.GPU_Coder_Interface_for_Deep_Learning +#product.GUIDE_to_App_Designer_Migration_Tool_for_MATLAB +#product.Geoid_Data_for_Aerospace_Toolbox +#product.HDL_Coder_Support_Package_for_Intel_FPGA_and_SoC_Devices +#product.HDL_Coder_Support_Package_for_Microchip_FPGA_and_SoC_Devices +#product.Hyperspectral_Imaging_Library_for_Image_Processing_Toolbox +#product.Image_Acquisition_Toolbox_Support_Package_for_DCAM_Hardware +#product.Image_Acquisition_Toolbox_Support_Package_for_GenICam_Interface +#product.Image_Acquisition_Toolbox_Support_Package_for_GigE_Vision_Hardware +#product.Image_Acquisition_Toolbox_Support_Package_for_Kinect_for_Windows_Sensor +#product.Image_Acquisition_Toolbox_Support_Package_for_Matrox_Hardware +#product.Image_Acquisition_Toolbox_Support_Package_for_National_Instruments_Frame_Grabbers +#product.Image_Acquisition_Toolbox_Support_Package_for_OS_Generic_Video_Interface +#product.Image_Acquisition_Toolbox_Support_Package_for_Point_Grey_Hardware +#product.Image_Acquisition_Toolbox_Support_Package_for_Teledyne_DALSA_Sapera_Hardware +#product.Image_Processing_Toolbox_Image_Data +#product.Image_Processing_Toolbox_Model_for_Circle_Detection +#product.Image_Processing_Toolbox_Model_for_Segment_Anything_Model +#product.Image_Processing_Toolbox_Model_for_Segment_Anything_Model_2 +#product.Instrument_Control_Toolbox_Support_Package_for_IVI_and_VXIplug&play_Drivers +#product.Instrument_Control_Toolbox_Support_Package_for_Keysight_(Agilent)_IO_Libraries_and_VISA_Interface +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-845x_I2C/SPI_Interface +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-DCPower_Power_Supplies +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-DMM_Digital_Multimeters +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-FGEN_Function_Generators +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-SCOPE_Oscilloscopes +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_NI-SWITCH_Hardware +#product.Instrument_Control_Toolbox_Support_Package_for_National_Instruments_VISA_and_ICP_Interfaces +#product.Instrument_Control_Toolbox_Support_Package_for_Rohde_Schwarz_VISA_Interface +#product.Instrument_Control_Toolbox_Support_Package_for_Total_Phase_Aardvark_I2C/SPI_Interface +#product.Integro-Differential_Modeling_Framework_for_MATLAB +#product.Large-Scale_Cloud_Simulation_for_Simulink +#product.MATLAB_Basemap_Data_-_bluegreen +#product.MATLAB_Basemap_Data_-_colorterrain +#product.MATLAB_Basemap_Data_-_grayland +#product.MATLAB_Basemap_Data_-_grayterrain +#product.MATLAB_Basemap_Data_-_landcover +#product.MATLAB_Client_for_MATLAB_Production_Server +#product.MATLAB_Coder_Interface_for_Deep_Learning +#product.MATLAB_Coder_Interface_for_Visual_Studio_Code_Debugging +#product.MATLAB_Coder_Support_Package_for_NVIDIA_Jetson_and_NVIDIA_DRIVE_Platforms +#product.MATLAB_Coder_Support_Package_for_PyTorch_and_LiteRT_Models +#product.MATLAB_Object_Performance_Improvement +#product.MATLAB_Support_Package_for_Android_Sensors +#product.MATLAB_Support_Package_for_Apple_iOS_Sensors +#product.MATLAB_Support_Package_for_Arduino_Hardware +#product.MATLAB_Support_Package_for_BeagleBone_Black_Hardware +#product.MATLAB_Support_Package_for_IP_Cameras +#product.MATLAB_Support_Package_for_LEGO_MINDSTORMS_EV3_Hardware +#product.MATLAB_Support_Package_for_Parrot_Drones +#product.MATLAB_Support_Package_for_Quantum_Computing +#product.MATLAB_Support_Package_for_Ryze_Tello_Drones +#product.MATLAB_Support_Package_for_USB_Webcams +#product.MATLAB_Support_for_HEIF/HEIC_Image_Format +#product.MATLAB_Support_for_MinGW-w64_C/C++/Fortran_Compiler +#product.MATLAB_Support_for_OpenJDK +#product.MATLAB_Support_for_Python_3.13 +#product.Machine_Learning_Models_for_Antenna_Toolbox +#product.MariaDB_ODBC_Driver_for_Database_Toolbox +#product.Medical_Imaging_Toolbox_Interface_for_Cellpose +#product.Medical_Imaging_Toolbox_Interface_for_MONAI_Label_Library +#product.Medical_Imaging_Toolbox_Interface_for_Whole_Slide_Imaging_File_Reader +#product.Medical_Imaging_Toolbox_Model_for_Medical_Segment_Anything_Model +#product.Mixed-Signal_Blockset_Models +#product.Modelscape_for_MATLAB +#product.Multi-Version_Co-Simulation_for_Simulink +#product.Optical_Design_and_Simulation_Library_for_Image_Processing_Toolbox +#product.Point_Cloud_Toolbox_Interface_for_OpenPCDet_Library +#product.Point_Cloud_Toolbox_Model_for_PointNet++_Classification +#product.Point_Cloud_Toolbox_Model_for_RandLA-Net_Semantic_Segmentation +#product.Point_Cloud_Toolbox_Model_for_SNAP_Segmentation_Network +#product.Point_Cloud_Toolbox_Support_Package_for_Hokuyo_LiDAR_Sensors +#product.Point_Cloud_Toolbox_Support_Package_for_Ouster_Lidar_Sensors +#product.Point_Cloud_Toolbox_Support_Package_for_SICK_Lidar_Sensors +#product.Point_Cloud_Toolbox_Support_Package_for_Velodyne_LiDAR_Sensors +#product.PostgreSQL_ODBC_Driver_for_Database_Toolbox +#product.Power_Converter_Control_with_Motor_Control_Blockset +#product.Powertrain_Blockset_Drive_Cycle_Data +#product.RF_Blockset_Models_for_Analog_Devices_RF_Transceivers +#product.ROS_Toolbox_Support_Package_for_TurtleBot-Based_Robots +#product.Radar_Toolbox_Support_Package_for_Texas_Instruments_mmWave_Radar_Sensors +#product.Reduced_Order_Modeler_for_MATLAB +#product.Robotics_System_Toolbox_Interface_for_Unreal_Engine_Projects +#product.Robotics_System_Toolbox_Offroad_Autonomy_Library +#product.Robotics_System_Toolbox_Robot_Library_Data +#product.Robotics_System_Toolbox_Support_Package_for_Kinova_Gen3_Manipulators +#product.Robotics_System_Toolbox_Support_Package_for_Universal_Robots_UR_Series_Manipulators +#product.Scenario_Builder_for_Automated_Driving_Toolbox +#product.SerDes_Toolbox_Support_Package_for_Linux_Cross_Compiler +#product.Simulink_3D_Animation_Interface_for_Unreal_Engine_Projects +#product.Simulink_Coder_Support_Package_for_ARM_Cortex-based_VEX_Microcontroller +#product.Simulink_Coder_Support_Package_for_BBC_microbit_Board +#product.Simulink_Coder_Support_Package_for_BeagleBone_Blue_Hardware +#product.Simulink_Coder_Support_Package_for_NXP_FRDM-K64F_Board +#product.Simulink_Coder_Support_Package_for_NXP_FRDM-KL25Z_Board +#product.Simulink_Coder_Support_Package_for_VEX_EDR_V5_Robot_Brain +#product.Simulink_Interface_for_Siemens_MF-Tyre/MF-Swift_Tire_Model +#product.Simulink_Real-Time_XIL_Support_Package +#product.Simulink_Support_Package_for_Android_Devices +#product.Simulink_Support_Package_for_Arduino_Hardware +#product.Simulink_Support_Package_for_LEGO_MINDSTORMS_EV3_Hardware +#product.Simulink_Support_Package_for_Parrot_Minidrones +#product.Simulink_Support_Package_for_Rust_Code +#product.Simulink_Test_Support_Package_for_ASAM_XIL +#product.SoC_Blockset_Support_Package_for_AMD_FPGA_and_SoC_Devices +#product.Software_Component_Designer_for_Simulink +#product.Streaming_Data_Framework_for_MATLAB_Production_Server +#product.Text_Analytics_Toolbox_Model_for_BERT-Base_Multilingual_Cased_Network +#product.Text_Analytics_Toolbox_Model_for_BERT-Base_Network +#product.Text_Analytics_Toolbox_Model_for_BERT-Large_Network +#product.Text_Analytics_Toolbox_Model_for_BERT-Mini_Network +#product.Text_Analytics_Toolbox_Model_for_BERT-Small_Network +#product.Text_Analytics_Toolbox_Model_for_BERT-Tiny_Network +#product.Text_Analytics_Toolbox_Model_for_all-MiniLM-L12-v2_Network +#product.Text_Analytics_Toolbox_Model_for_all-MiniLM-L6-v2_Network +#product.Text_Analytics_Toolbox_Model_for_fastText_English_16_Billion_Token_Word_Embedding +#product.Text_Analytics_Toolbox_Model_from_UDify_Data +#product.Third-Party_Tool_Connector_for_Requirements_Toolbox +#product.Time_Series_Anomaly_Detection_for_MATLAB +#product.UAV_Toolbox_Interface_for_Unreal_Engine_Projects +#product.UAV_Toolbox_Support_Package_for_ArduPilot_Autopilots +#product.UAV_Toolbox_Support_Package_for_PX4_Autopilots +#product.Vehicle_Dynamics_Blockset_Interface_for_Unreal_Engine_Projects +#product.Vehicle_Dynamics_Blockset_Maneuver_Data +#product.Visual_Inspection_Toolbox_Model_for_CounTR_Object_Counting +#product.Visual_Inspection_Toolbox_Model_for_Student-Teacher_Anomaly_Detection +#product.Visual_Inspection_Toolbox_Model_for_YOLOX_Object_Detection +#product.WINNER_II_Channel_Model_for_Communications_Toolbox +#product.Wireless_Testbench_Support_Package_for_NI_USRP_Radios + + +######################################################################## +## OPTIONAL FEATURES +######################################################################## +## +## Uncomment the lines for the optional features you want to install or download. + +#product.3D_Scenarios +#product.Airport_Scene +#product.Apple_Hill_Scene +#product.Construction_Site_Scene +#product.Curved_Road_Scene +#product.Double_Lane_Change_Scene +#product.Earth_Scene +#product.Empty_Grass_Scene +#product.Empty_Scene +#product.Griffiss_Airport_Scene +#product.Large_Parking_Lot_Scene +#product.Moon_Scene +#product.Multi-Sensor_Calibration_Tools +#product.NVIDIA_TensorRT +#product.Offroad_Pit_Mining_Scene +#product.Open_Surface_Scene +#product.Parking_Lot_Scene +#product.Room_Acoustics_Simulation +#product.Simscape_Acoustics +#product.Space_Scene +#product.Straight_Road_Scene +#product.Suburban_Scene +#product.US_City_Block_Scene +#product.US_Highway_Scene +#product.Underwater_Acoustic_Channels +#product.Underwater_Scene +#product.Virtual_MCity_Scene +#product.ZalaZONE_Automotive_Proving_Ground_High-speed_Handling_Course_Scene +#product.ZalaZONE_Automotive_Proving_Ground_Hill_Tracks_Scene +#product.ZalaZONE_Automotive_Proving_Ground_Smart_City_Scene + + +######################################################################## +## CHECKSUM +######################################################################## +## +## NOTE: DO NOT edit this field. MathWorks uses this field to +## check the integrity of the input file. Changing the value +## of the checksum field invalidates this input file. + +?checksum=UjIwMjZi diff --git a/addons/osfstorage/decorators.py b/addons/osfstorage/decorators.py index decf4489ee5..b946976f5a8 100644 --- a/addons/osfstorage/decorators.py +++ b/addons/osfstorage/decorators.py @@ -105,7 +105,8 @@ def wrapped(payload, *args, **kwargs): 'source': source, 'destination': dest_parent, 'name': payload['destination']['name'], - 'is_check_permission': is_check_permission + 'is_check_permission': is_check_permission, + 'replaced_size': int(payload.get('replaced_size', 0)), }) except KeyError: raise HTTPError(http_status.HTTP_400_BAD_REQUEST) diff --git a/addons/osfstorage/models.py b/addons/osfstorage/models.py index 8786699ea9d..53813d82a72 100644 --- a/addons/osfstorage/models.py +++ b/addons/osfstorage/models.py @@ -361,6 +361,7 @@ def serialize(self, include_full=None, version=None): earliest_version = self.versions.order_by('created').first() ret.update({ 'version': self.versions.count(), + 'size': version.size if version else ret.get('size'), 'md5': version.metadata.get('md5') if version else None, 'sha256': version.metadata.get('sha256') if version else None, 'sha512': version.metadata.get('sha512') if version else None, diff --git a/addons/osfstorage/tests/test_models.py b/addons/osfstorage/tests/test_models.py index 27d81fc5b9b..fbaa3eb492e 100644 --- a/addons/osfstorage/tests/test_models.py +++ b/addons/osfstorage/tests/test_models.py @@ -137,6 +137,53 @@ def test_serialize(self): u'sha512': None, }) + def test_serialize_size_by_version(self): + file = OsfStorageFile(name='versioned_file.txt', target=self.node_settings.owner) + file.save() + + version1 = file.create_version( + self.user, + { + u'service': u'cloud', + settings.WATERBUTLER_RESOURCE: u'osf', + u'object': u'aaa111', + }, { + u'size': 1000, + u'contentType': u'text/plain', + }) + + version2 = file.create_version( + self.user, + { + u'service': u'cloud', + settings.WATERBUTLER_RESOURCE: u'osf', + u'object': u'bbb222', + }, { + u'size': 5000, + u'contentType': u'text/plain', + }) + + # Default serialize (no version arg) returns latest version's size + result_default = file.serialize() + assert_equal(result_default[u'size'], 5000) + assert_equal(result_default[u'version'], 2) + assert_equal(result_default[u'modified'], version2.created.isoformat()) + assert_equal(result_default[u'created'], version1.created.isoformat()) + + # Serialize with version=1 returns version 1's size + result_v1 = file.serialize(version=1) + assert_equal(result_v1[u'size'], 1000) + assert_equal(result_v1[u'version'], 2) + assert_equal(result_v1[u'modified'], version1.created.isoformat()) + assert_equal(result_v1[u'created'], version1.created.isoformat()) + + # Serialize with version=2 returns version 2's size + result_v2 = file.serialize(version=2) + assert_equal(result_v2[u'size'], 5000) + assert_equal(result_v2[u'version'], 2) + assert_equal(result_v2[u'modified'], version2.created.isoformat()) + assert_equal(result_v2[u'created'], version1.created.isoformat()) + def test_get_child_by_name(self): child = self.node_settings.get_root().append_file('Test') assert_equal(child, self.node_settings.get_root().find_child_by_name('Test')) diff --git a/addons/osfstorage/tests/test_views_copy_move_quota.py b/addons/osfstorage/tests/test_views_copy_move_quota.py new file mode 100644 index 00000000000..4948d7c8770 --- /dev/null +++ b/addons/osfstorage/tests/test_views_copy_move_quota.py @@ -0,0 +1,1323 @@ +# encoding: utf-8 +""" +Tests for new quota and FileInfo logic added to addons/osfstorage/views.py. + +Covers: + - _get_all_descendant_file_ids + - osfstorage_copy_hook (file + folder, FileInfo UPSERT, quota delta) + - osfstorage_move_hook (file + folder, intra-target replaced_size, inter-target quota) + - osfstorage_create_child (FileInfo UPSERT + quota delta on upload) + - osfstorage_delete (folder quota subtraction before deletion) + - TestUploadIntegration (full upload flow: create_child -> create_waterbutler_log) +""" +from __future__ import unicode_literals + +import time as time_module +import mock +import pytest +from nose.tools import assert_equal, assert_true, assert_false + +from framework.auth import signing +from website.util import api_url_for + +from addons.osfstorage.tests.utils import StorageTestCase, make_payload +from addons.osfstorage.tests import factories +from addons.osfstorage.tests.utils import recursively_create_file + +from osf.models import FileInfo, BaseFileNode +from osf_tests.factories import ProjectFactory + + +# --------------------------------------------------------------------------- +# Helper +# --------------------------------------------------------------------------- + +def _sign(payload): + """Return a signed copy of *payload* using the default signer.""" + return signing.sign_data(signing.default_signer, payload) + + +# --------------------------------------------------------------------------- +# Test helper: _get_all_descendant_file_ids +# --------------------------------------------------------------------------- + +@pytest.mark.django_db +class TestGetAllDescendantFileIds(StorageTestCase): + """Unit-tests for the private helper _get_all_descendant_file_ids.""" + + def _call(self, folder_node): + from addons.osfstorage.views import _get_all_descendant_file_ids + return _get_all_descendant_file_ids(folder_node) + + def test_empty_folder_returns_empty_list(self): + """An empty root folder (no children) must return [].""" + root = self.node_settings.get_root() + result = self._call(root) + assert_equal(result, []) + + def test_flat_folder_with_files(self): + """Files directly under a folder must all be returned.""" + root = self.node_settings.get_root() + f1 = root.append_file('alpha.txt') + f2 = root.append_file('beta.txt') + result = self._call(root) + assert set(result) == {f1.id, f2.id} + + def test_nested_folders_return_all_files(self): + """ + Structure: + root/ + sub1/ + deep.txt + file.txt + Both files must be returned, folder IDs must NOT be included. + """ + root = self.node_settings.get_root() + sub1 = root.append_folder('sub1') + deep = sub1.append_file('deep.txt') + flat = root.append_file('file.txt') + result = self._call(root) + assert set(result) == {deep.id, flat.id} + + def test_trashed_files_excluded(self): + """TrashedFile/TrashedFolder must not appear in the result.""" + root = self.node_settings.get_root() + live = root.append_file('live.txt') + dead = root.append_file('dead.txt') + # Soft-delete the file + dead.delete(user=self.user) + result = self._call(root) + assert live.id in result + assert dead.id not in result + + def test_only_osfstorage_provider(self): + """Files from other providers must be excluded (provider filter).""" + root = self.node_settings.get_root() + # append_file creates osfstorage files by default – just verify the + # provider guard does not block those + f = root.append_file('normal.txt') + result = self._call(root) + assert f.id in result + + +# --------------------------------------------------------------------------- +# Helpers shared by copy/move hook tests +# --------------------------------------------------------------------------- + +class HookTestBase(StorageTestCase): + """Base class providing signed hook helpers used by copy/move tests.""" + + def _post_hook(self, view_name, guid, payload): + return self.app.post_json( + api_url_for(view_name, guid=guid, **_sign(payload)), + {}, + expect_errors=True, + ) + + def _send_hook(self, view_name, guid_target, payload, target_node=None): + """Send a signed POST to *view_name* and return the response.""" + target_node = target_node or self.node + return self.app.post_json( + api_url_for(view_name, guid=guid_target), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + +# --------------------------------------------------------------------------- +# osfstorage_copy_hook – file copy +# --------------------------------------------------------------------------- +@pytest.mark.enable_implicit_clean +@pytest.mark.django_db +class TestCopyHookFileQuota(StorageTestCase): + """ + Tests for the FILE branch of osfstorage_copy_hook. + + New logic (views.py lines 159-208): + • Creates / updates a FileInfo record for the cloned file. + • Computes delta = new_size - replaced_size and calls update_quota. + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + def _copy_file(self, source_file, dest_folder, dest_target=None, replaced_size=0): + dest_target = dest_target or self.node + payload = { + 'source': source_file._id, + 'target': self.root_node._id, + 'user': self.user._id, + 'destination': { + 'parent': dest_folder._id, + 'target': dest_target._id, + 'name': source_file.name, + }, + } + if replaced_size: + payload['replaced_size'] = str(replaced_size) + # signed payload must go in the request BODY, not URL params + return self.app.post_json( + api_url_for('osfstorage_copy_hook', guid=self.node._id), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_file_creates_fileinfo(self, mock_st, mock_uq): + """After a file copy, a FileInfo record must exist for the clone.""" + version = factories.FileVersionFactory(size=500) + src = self.root_node.append_file('src.txt') + src.add_version(version) + src.save() + + dest_folder = self.root_node.append_folder('dest') + resp = self._copy_file(src, dest_folder) + assert resp.status_code == 201, f'copy hook returned {resp.status_code}: {resp.json}' + + # Exactly one FileInfo must exist for the cloned file (not the source) + cloned = dest_folder.find_child_by_name('src.txt') + assert cloned is not None + fi = FileInfo.objects.filter(file=cloned).first() + assert fi is not None + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_file_quota_delta_positive(self, mock_st, mock_uq): + """ + When a file is copied WITHOUT a replaced file (replaced_size=0), + the full new_size is added to the quota. + """ + version = factories.FileVersionFactory(size=300) + src = self.root_node.append_file('data.bin') + src.add_version(version) + src.save() + + dest_folder = self.root_node.append_folder('out') + resp = self._copy_file(src, dest_folder, replaced_size=0) + assert resp.status_code == 201, f'copy hook returned {resp.status_code}: {resp.json}' + + # update_quota must have been called with add=True and the correct size + assert mock_uq.called + call_kwargs = mock_uq.call_args + # delta > 0 → add=True + assert call_kwargs[1].get('add', call_kwargs[0][-1]) is True + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_file_no_quota_update_when_delta_zero(self, mock_st, mock_uq): + """ + When new_size == replaced_size the delta is 0, so update_quota must + NOT be called (views.py line 203: if delta != 0). + """ + version = factories.FileVersionFactory(size=100) + src = self.root_node.append_file('equal.txt') + src.add_version(version) + src.save() + + dest_folder = self.root_node.append_folder('same') + self._copy_file(src, dest_folder, replaced_size=100) + + assert not mock_uq.called + + +# --------------------------------------------------------------------------- +# osfstorage_copy_hook – folder copy +# --------------------------------------------------------------------------- + +@pytest.mark.enable_implicit_clean +@pytest.mark.django_db +class TestCopyHookFolderQuota(StorageTestCase): + """ + Tests for the FOLDER branch of osfstorage_copy_hook. + + New logic (views.py lines 165-198): + • Builds a source_size_map from FileInfo records (or falls back to + versions) for every descendant. + • Uses bulk_create (with IntegrityError fallback) to create FileInfo + records for the cloned folder's descendants. + • Forces replaced_size = 0 because osfstorage_delete has already + subtracted it. + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + def _copy_folder(self, source_folder, dest_folder, dest_target=None): + dest_target = dest_target or self.node + payload = { + 'source': source_folder._id, + 'target': self.root_node._id, + 'user': self.user._id, + 'destination': { + 'parent': dest_folder._id, + 'target': dest_target._id, + 'name': source_folder.name, + }, + } + # signed payload must go in the request BODY, not URL params + return self.app.post_json( + api_url_for('osfstorage_copy_hook', guid=self.node._id), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_folder_creates_fileinfo_for_descendants(self, mock_st, mock_uq): + """ + After copying a folder, every descendant FILE of the clone must have + a corresponding FileInfo record. + """ + # Build source folder with two files + src_folder = self.root_node.append_folder('src_folder') + v1 = factories.FileVersionFactory(size=200) + f1 = src_folder.append_file('one.txt') + f1.add_version(v1); f1.save() + + v2 = factories.FileVersionFactory(size=400) + f2 = src_folder.append_file('two.txt') + f2.add_version(v2); f2.save() + + # Pre-create FileInfo so source_size_map is populated + FileInfo.objects.update_or_create(file=f1, defaults={'file_size': 200}) + FileInfo.objects.update_or_create(file=f2, defaults={'file_size': 400}) + + dest_folder = self.root_node.append_folder('dest_folder') + resp = self._copy_folder(src_folder, dest_folder) + assert resp.status_code == 201, f'copy hook returned {resp.status_code}: {resp.json}' + + cloned = dest_folder.find_child_by_name('src_folder') + assert cloned is not None + + from addons.osfstorage.views import _get_all_descendant_file_ids + cloned_ids = _get_all_descendant_file_ids(cloned) + assert len(cloned_ids) == 2 + for fid in cloned_ids: + assert FileInfo.objects.filter(file_id=fid).exists(), \ + f'FileInfo missing for cloned file id={fid}' + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_folder_fallback_when_fileinfo_missing(self, mock_st, mock_uq): + """ + If the source file has no FileInfo record, the code falls back to + the latest FileVersion.size. The clone must still get a FileInfo + record (possibly size=0 if no version exists either). + """ + src_folder = self.root_node.append_folder('no_fi_folder') + v = factories.FileVersionFactory(size=111) + f = src_folder.append_file('nofi.txt') + f.add_version(v); f.save() + # Intentionally do NOT create a FileInfo for f + + dest_folder = self.root_node.append_folder('dest_nofi') + self._copy_folder(src_folder, dest_folder) + + cloned = dest_folder.find_child_by_name('no_fi_folder') + from addons.osfstorage.views import _get_all_descendant_file_ids + cloned_ids = _get_all_descendant_file_ids(cloned) + # The clone must have a FileInfo record + for fid in cloned_ids: + assert FileInfo.objects.filter(file_id=fid).exists() + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.osfstorage.views.logger') + def test_copy_folder_fallback_none_size_logs_warning_and_sets_zero(self, mock_logger, mock_st, mock_uq): + """ + When the source file has no FileInfo + AND its latest version has size=None, the clone's FileInfo must be 0 + and a warning must be logged. + """ + src_folder = self.root_node.append_folder('none_size_folder') + v = factories.FileVersionFactory(size=None) + f = src_folder.append_file('none_size.txt') + f.add_version(v); f.save() + # Intentionally do NOT create a FileInfo for f + + dest_folder = self.root_node.append_folder('dest_none_size') + resp = self._copy_folder(src_folder, dest_folder) + assert resp.status_code == 201 + + cloned = dest_folder.find_child_by_name('none_size_folder') + from addons.osfstorage.views import _get_all_descendant_file_ids + cloned_ids = _get_all_descendant_file_ids(cloned) + for fid in cloned_ids: + fi = FileInfo.objects.filter(file_id=fid).first() + assert fi is not None + assert fi.file_size == 0 + assert mock_logger.warning.called + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.osfstorage.views.logger') + def test_copy_folder_fallback_negative_size_logs_warning_and_sets_zero(self, mock_logger, mock_st, mock_uq): + """ A negative version size (-1 ="not yet set") must be treated as 0 with a warning """ + src_folder = self.root_node.append_folder('neg_size_folder') + v = factories.FileVersionFactory(size=-1) + f = src_folder.append_file('neg_size.txt') + f.add_version(v); f.save() + # Intentionally do NOT create a FileInfo for f + + dest_folder = self.root_node.append_folder('dest_neg_size') + resp = self._copy_folder(src_folder, dest_folder) + assert resp.status_code == 201 + + cloned = dest_folder.find_child_by_name('neg_size_folder') + from addons.osfstorage.views import _get_all_descendant_file_ids + cloned_ids = _get_all_descendant_file_ids(cloned) + for fid in cloned_ids: + fi = FileInfo.objects.filter(file_id=fid).first() + assert fi is not None + assert fi.file_size == 0 + assert mock_logger.warning.called + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.osfstorage.views._get_all_descendant_file_ids') + def test_copy_folder_none_copied_from_id_sets_size_zero(self, mock_descendants, mock_st, mock_uq): + """ + When a cloned file has copied_from_id=None (not in source_size_map), + the else branch (views.py: 'src_id is None and not in map') sets size=0 + and still creates a FileInfo record for that file. + """ + src_folder = self.root_node.append_folder('src_none_id') + v = factories.FileVersionFactory(size=100) + src_file = src_folder.append_file('src_none.txt') + src_file.add_version(v) + src_file.save() + FileInfo.objects.update_or_create(file=src_file, defaults={'file_size': 100}) + + dest_folder = self.root_node.append_folder('dest_none_id') + # A fresh file with no copied_from → copied_from_id will be None in the DB + orphan_file = dest_folder.append_file('orphan.txt') + orphan_file.save() + + # First call → source descendants; second call → cloned descendants (orphan) + mock_descendants.side_effect = [ + [src_file.id], # _get_all_descendant_file_ids(source) + [orphan_file.id], # _get_all_descendant_file_ids(cloned) + ] + + resp = self._copy_folder(src_folder, dest_folder) + assert resp.status_code == 201 + + # The else branch must have produced size=0 → FileInfo.file_size == 0 + fi = FileInfo.objects.filter(file=orphan_file).first() + assert fi is not None, 'FileInfo must be created even when copied_from_id is None' + assert fi.file_size == 0 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_copy_folder_bulk_create_integrity_error_fallback(self, mock_st, mock_uq): + """ + When FileInfo.objects.bulk_create raises IntegrityError the except-block + must call get_or_create for every FileInfo in file_infos. + """ + from django.db import IntegrityError as DjangoIntegrityError + + src_folder = self.root_node.append_folder('src_ie_folder') + v = factories.FileVersionFactory(size=75) + src_file = src_folder.append_file('ie_file.txt') + src_file.add_version(v) + src_file.save() + FileInfo.objects.update_or_create(file=src_file, defaults={'file_size': 75}) + + dest_folder = self.root_node.append_folder('dest_ie_folder') + + original_get_or_create = FileInfo.objects.get_or_create + with mock.patch.object( + FileInfo.objects, 'bulk_create', + side_effect=DjangoIntegrityError('duplicate key') + ): + with mock.patch.object( + FileInfo.objects, 'get_or_create', + wraps=original_get_or_create + ) as spy_goc: + resp = self._copy_folder(src_folder, dest_folder) + + assert resp.status_code == 201 + assert spy_goc.called, ( + 'get_or_create must be called as the IntegrityError fallback' + ) + # One get_or_create call per cloned descendant file + from addons.osfstorage.views import _get_all_descendant_file_ids + cloned = dest_folder.find_child_by_name('src_ie_folder') + cloned_ids = _get_all_descendant_file_ids(cloned) + assert spy_goc.call_count == len(cloned_ids) + + +# --------------------------------------------------------------------------- +# osfstorage_move_hook – quota logic +# --------------------------------------------------------------------------- + +@pytest.mark.django_db +class TestMoveHookQuota(StorageTestCase): + """ + Tests for the quota-related additions in osfstorage_move_hook. + + New logic (views.py lines 231-258): + FILE move: + • FileInfo UPSERT (file=source, file_size=latest_version.size). + FOLDER move: + • Aggregates FileInfo for all descendants. + • replaced_size = 0 (osfstorage_delete already subtracted). + QUOTA UPDATE: + • inter-target: subtract from source, add delta to dest. + • intra-target with replaced_size > 0: subtract replaced_size only. + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + def _move(self, source, dest_folder, dest_target=None, replaced_size=None, + is_check_permission=None): + dest_target = dest_target or self.node + payload = { + 'source': source._id, + 'target': self.root_node._id, + 'user': self.user._id, + 'destination': { + 'parent': dest_folder._id, + 'target': dest_target._id, + 'name': source.name, + }, + } + if replaced_size is not None: + payload['replaced_size'] = str(replaced_size) + if is_check_permission is not None: + payload['is_check_permission'] = is_check_permission + # signed payload must go in the request BODY, not URL params + return self.app.post_json( + api_url_for('osfstorage_move_hook', guid=self.node._id), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + # --- FILE MOVE --- + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_file_upserts_fileinfo(self, mock_st, mock_uq): + """ + Moving a FILE must create/update a FileInfo record for that file + reflecting the latest version size. + """ + version = factories.FileVersionFactory(size=250) + src = self.root_node.append_file('move_me.txt') + src.add_version(version); src.save() + + dest_folder = self.root_node.append_folder('landing') + self._move(src, dest_folder) + + src.reload() + fi = FileInfo.objects.filter(file=src).first() + assert fi is not None + assert fi.file_size == 250 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_file_inter_target_subtracts_source_quota(self, mock_st, mock_uq): + """ + Moving a FILE to a *different* target node must call update_quota + on the source target with add=False. + """ + # Different creator: keeps this test on the "different UserQuota record" + # branch of osfstorage_move_hook, distinct from the same-creator tests below. + other_project = ProjectFactory() + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('remote') + + version = factories.FileVersionFactory(size=150) + src = self.root_node.append_file('cross.txt') + src.add_version(version); src.save() + + self._move(src, dest_folder, dest_target=other_project) + + # At least one call to update_quota with add=False (source deduction) + assert mock_uq.called + subtract_calls = [c for c in mock_uq.call_args_list + if c[1].get('add', c[0][-1]) is False] + assert len(subtract_calls) >= 1 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_file_intra_target_replaced_size_subtracted(self, mock_st, mock_uq): + """ + Moving a FILE within the SAME target node and providing replaced_size > 0 + must subtract replaced_size from the target (views.py line 255-257). + """ + version = factories.FileVersionFactory(size=100) + src = self.root_node.append_file('replace_target.txt') + src.add_version(version); src.save() + + dest_folder = self.root_node.append_folder('same_target_dest') + self._move(src, dest_folder, replaced_size=80) + + # update_quota must have been called (for the replace deduction) + assert mock_uq.called + subtract_calls = [c for c in mock_uq.call_args_list + if c[1].get('add', c[0][-1]) is False] + assert len(subtract_calls) >= 1 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_file_intra_target_no_replace_no_quota_update(self, mock_st, mock_uq): + """ + Moving a FILE within the SAME target and replaced_size=0 must NOT + trigger any update_quota call. + """ + version = factories.FileVersionFactory(size=100) + src = self.root_node.append_file('same_no_replace.txt') + src.add_version(version); src.save() + + dest_folder = self.root_node.append_folder('inner') + self._move(src, dest_folder, replaced_size=0) + + assert not mock_uq.called + + # --- FOLDER MOVE --- + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_folder_aggregates_fileinfo(self, mock_st, mock_uq): + """ + Moving a FOLDER must aggregate FileInfo sizes for all descendants + (new_size) and pass that to update_quota on the source (add=False). + """ + # Different creator: keeps this test on the "different UserQuota record" + # branch of osfstorage_move_hook, distinct from the same-creator tests below. + other_project = ProjectFactory() + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('folder_dest') + + src_folder = self.root_node.append_folder('move_folder') + v1 = factories.FileVersionFactory(size=100) + f1 = src_folder.append_file('a.txt'); f1.add_version(v1); f1.save() + v2 = factories.FileVersionFactory(size=200) + f2 = src_folder.append_file('b.txt'); f2.add_version(v2); f2.save() + + FileInfo.objects.update_or_create(file=f1, defaults={'file_size': 100}) + FileInfo.objects.update_or_create(file=f2, defaults={'file_size': 200}) + + self._move(src_folder, dest_folder, dest_target=other_project) + + # update_quota must have been called for source deduction (300 total) + assert mock_uq.called + subtract_calls = [c for c in mock_uq.call_args_list + if c[1].get('add', c[0][-1]) is False] + assert len(subtract_calls) >= 1 + # The size passed must match the aggregate (300) + sizes = [c[0][1] for c in subtract_calls] + assert 300 in sizes + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_move_folder_replaced_size_zeroed(self, mock_st, mock_uq): + """ + When moving a folder, replaced_size must be set to 0 before computing + delta (osfstorage_delete already subtracted it). Therefore, if the + caller passes a non-zero replaced_size for a folder move the effective + delta equals new_size, NOT new_size - replaced_size. + """ + # Different creator: keeps this test on the "different UserQuota record" + # branch of osfstorage_move_hook, distinct from the same-creator tests below. + other_project = ProjectFactory() + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('folder_dest2') + + src_folder = self.root_node.append_folder('move_folder2') + v = factories.FileVersionFactory(size=300) + f = src_folder.append_file('c.txt'); f.add_version(v); f.save() + FileInfo.objects.update_or_create(file=f, defaults={'file_size': 300}) + + # Pass a non-zero replaced_size – it must be ignored for folders + self._move(src_folder, dest_folder, dest_target=other_project, replaced_size=999) + + assert mock_uq.called + # The delta added to dest must be 300 (not 300-999=-699) + add_calls = [c for c in mock_uq.call_args_list + if c[1].get('add', c[0][-1]) is True] + sizes_added = [c[0][1] for c in add_calls] + assert 300 in sizes_added + + # --- SAME-CREATOR CROSS-NODE MOVE (branch by UserQuota record, not node) --- + # These exercise osfstorage_move_hook against a REAL UserQuota row (update_quota is + # not mocked) because the bug being fixed is a real arithmetic corruption of `used`, + # not just which mock gets called. + + def test_move_file_to_other_node_same_creator_does_not_clamp_used_to_zero(self): + """A move to a different node owned by the same creator shares one UserQuota row; + `used` must not clamp to 0 between the two opposing writes on that row.""" + from osf.models import UserQuota + + UserQuota.objects.get_or_create( + user=self.user, storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 10000, 'used': 100}, + ) + + other_project = ProjectFactory(creator=self.user) + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('same_creator_dest') + + version = factories.FileVersionFactory(size=500) + src = self.root_node.append_file('same_creator.txt') + src.add_version(version); src.save() + + resp = self._move(src, dest_folder, dest_target=other_project, replaced_size=0) + assert resp.status_code == 200 + + used = UserQuota.objects.get(user=self.user, storage_type=UserQuota.NII_STORAGE).used + assert used == 100 + + def test_move_file_to_other_node_same_creator_with_replace_subtracts_only_replaced(self): + """Same as above, but the destination overwrites a 50-byte item: net delta is -50.""" + from osf.models import UserQuota + + UserQuota.objects.get_or_create( + user=self.user, storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 10000, 'used': 100}, + ) + + other_project = ProjectFactory(creator=self.user) + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('same_creator_dest_replace') + + version = factories.FileVersionFactory(size=500) + src = self.root_node.append_file('same_creator_replace.txt') + src.add_version(version); src.save() + + resp = self._move(src, dest_folder, dest_target=other_project, replaced_size=50) + assert resp.status_code == 200 + + used = UserQuota.objects.get(user=self.user, storage_type=UserQuota.NII_STORAGE).used + assert used == 50 + + def test_move_file_to_other_node_different_creator_still_transfers_full_size(self): + """A move between nodes with different creators still hits two distinct UserQuota + rows and transfers the full size.""" + from osf.models import UserQuota + + UserQuota.objects.get_or_create( + user=self.user, storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 10000, 'used': 1000}, + ) + other_project = ProjectFactory() + UserQuota.objects.get_or_create( + user=other_project.creator, storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 10000, 'used': 0}, + ) + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('different_creator_dest') + + version = factories.FileVersionFactory(size=500) + src = self.root_node.append_file('different_creator.txt') + src.add_version(version); src.save() + + resp = self._move(src, dest_folder, dest_target=other_project, replaced_size=0) + assert resp.status_code == 200 + + src_used = UserQuota.objects.get(user=self.user, storage_type=UserQuota.NII_STORAGE).used + dest_used = UserQuota.objects.get( + user=other_project.creator, storage_type=UserQuota.NII_STORAGE + ).used + assert src_used == 500 + assert dest_used == 500 + + def test_move_folder_to_other_node_same_creator_is_quota_neutral(self): + """A folder move within the same UserQuota record results in no quota change and + no UserQuota.save() call.""" + from osf.models import UserQuota + + UserQuota.objects.get_or_create( + user=self.user, storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 10000, 'used': 100}, + ) + + other_project = ProjectFactory(creator=self.user) + other_root = other_project.get_addon('osfstorage').get_root() + dest_folder = other_root.append_folder('same_creator_folder_dest') + + src_folder = self.root_node.append_folder('same_creator_move_folder') + v = factories.FileVersionFactory(size=500) + f = src_folder.append_file('d.txt'); f.add_version(v); f.save() + FileInfo.objects.update_or_create(file=f, defaults={'file_size': 500}) + + with mock.patch.object(UserQuota, 'save') as mock_save: + resp = self._move(src_folder, dest_folder, dest_target=other_project, replaced_size=999) + assert resp.status_code == 200 + mock_save.assert_not_called() + + used = UserQuota.objects.get(user=self.user, storage_type=UserQuota.NII_STORAGE).used + assert used == 100 + + +# --------------------------------------------------------------------------- +# osfstorage_create_child – FileInfo UPSERT + quota delta +# --------------------------------------------------------------------------- + +@pytest.mark.django_db +class TestCreateChildQuota(StorageTestCase): + """ + Tests for the FileInfo / quota logic added to osfstorage_create_child. + + New logic (views.py lines 489-504): + • Computes new_size from the new FileVersion. + • Reads old_size from existing FileInfo (or 0 if none). + • UPSERT FileInfo with new_size. + • Calls update_quota(target, abs(delta), storage_type, add=(delta > 0)) + only when delta != 0. + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + def _upload(self, parent, name, size=123, unique_hash=None, target=None): + """ + Build and POST an upload payload. + + *unique_hash*: when provided, overrides ``metadata['name']`` so that + each call produces a distinct location-hash and is NOT treated as a + duplicate by ``FileVersion.is_duplicate``. Pass a different string on + each call to simulate a genuine new version. + """ + target = target or self.project + payload = make_payload(user=self.user, name=name) + payload['metadata']['size'] = size + if unique_hash is not None: + # Changing metadata['name'] changes the location['object'] value + # which changes the location_hash → not a duplicate + payload['metadata']['name'] = unique_hash + return self.app.post_json( + api_url_for( + 'osfstorage_create_child', + fid=parent._id, + guid=target._id, + ), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_upload_new_file_creates_fileinfo(self, mock_st, mock_uq): + """First upload of a file must create a FileInfo record with the correct size.""" + res = self._upload(self.root_node, 'brand_new.txt', size=500) + assert res.status_code in (200, 201) + record = self.root_node.find_child_by_name('brand_new.txt') + fi = FileInfo.objects.filter(file=record).first() + assert fi is not None + assert fi.file_size == 500 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_upload_new_file_calls_update_quota(self, mock_st, mock_uq): + """First upload (old_size=0) must call update_quota with add=True.""" + res = self._upload(self.root_node, 'new_quota.txt', size=300) + assert res.status_code in (200, 201) + assert mock_uq.called + call_kwargs = mock_uq.call_args + assert call_kwargs[1].get('add', call_kwargs[0][-1]) is True + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_upload_update_file_updates_fileinfo(self, mock_st, mock_uq): + """ + Re-uploading (new version) of the same file must update the FileInfo + to the new size. + + We pass unique_hash on each call so that the two uploads produce + different location hashes and are NOT treated as duplicates. Without + this, create_version returns the existing version (size=100) even when + we request size=200. + """ + # First upload + self._upload(self.root_node, 'versioned.txt', size=100, unique_hash='hash_v1') + record = self.root_node.find_child_by_name('versioned.txt') + + # Second upload – new hash forces a real new version with size=200 + mock_uq.reset_mock() + self._upload(self.root_node, 'versioned.txt', size=200, unique_hash='hash_v2') + record.reload() + fi = FileInfo.objects.filter(file=record).first() + assert fi is not None + assert fi.file_size == 200 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_upload_update_file_quota_delta(self, mock_st, mock_uq): + """ + When a file is updated with a genuinely new version, delta = new_size - old_size. + update_quota must be called with abs(delta) and add=True when size increases. + """ + # First upload sets old_size = 100 + self._upload(self.root_node, 'delta_test.txt', size=100, unique_hash='hash_a') + mock_uq.reset_mock() + + # Second upload: unique hash → real new version, new_size=250, delta=150 + self._upload(self.root_node, 'delta_test.txt', size=250, unique_hash='hash_b') + + assert mock_uq.called + call_kwargs = mock_uq.call_args + size_arg = call_kwargs[0][1] + add_arg = call_kwargs[1].get('add', call_kwargs[0][-1]) + assert size_arg == 150 + assert add_arg is True + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_upload_same_size_no_quota_update(self, mock_st, mock_uq): + """ + If the new version has the same size as the old version, delta=0 and + update_quota must NOT be called a second time. + """ + self._upload(self.root_node, 'same_size.txt', size=100) + mock_uq.reset_mock() + + # Upload again with the same size + self._upload(self.root_node, 'same_size.txt', size=100) + + # delta == 0 → update_quota should NOT be called + assert not mock_uq.called + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.osfstorage.views.logger') + def test_upload_none_size_logs_warning_and_sets_zero(self, mock_logger, mock_st, mock_uq): + """ + When WaterButler reports size=None for a new upload, FileInfo.file_size + must be 0 and a warning must be logged. + """ + res = self._upload(self.root_node, 'none_size.txt', size=None) + assert res.status_code in (200, 201) + record = self.root_node.find_child_by_name('none_size.txt') + fi = FileInfo.objects.filter(file=record).first() + assert fi is not None + assert fi.file_size == 0 + assert mock_logger.warning.called + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.osfstorage.views.logger') + def test_upload_negative_size_logs_warning_and_sets_zero(self, mock_logger, mock_st, mock_uq): + """ + A negative size (-1 = "not yet set") must be treated as 0 with a + warning, not stored as-is in FileInfo.file_size. + """ + res = self._upload(self.root_node, 'neg_size.txt', size=-1) + assert res.status_code in (200, 201) + record = self.root_node.find_child_by_name('neg_size.txt') + fi = FileInfo.objects.filter(file=record).first() + assert fi is not None + assert fi.file_size == 0 + assert mock_logger.warning.called + + +# --------------------------------------------------------------------------- +# osfstorage_delete – folder quota subtraction +# --------------------------------------------------------------------------- + +@pytest.mark.django_db +class TestDeleteHookFolderQuota(StorageTestCase): + """ + Tests for the folder-quota logic added to osfstorage_delete. + + New logic (views.py lines 533-542): + • Collects _get_all_descendant_file_ids for the folder. + • Sums FileInfo.file_size for those files. + • Zeroes the FileInfo records BEFORE deletion (race-condition guard). + • Calls update_quota(target, total_size, storage_type, add=False). + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + def _delete(self, file_node): + payload = {'user': self.user._id} + return self.app.delete( + '{url}?payload={payload}&signature={signature}'.format( + url=api_url_for( + 'osfstorage_delete', + guid=self.node._id, + fid=file_node._id, + ), + **signing.sign_data(signing.default_signer, payload) + ), + expect_errors=True, + ) + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_delete_folder_subtracts_quota(self, mock_st, mock_uq): + """ + Deleting a folder with 2 files (total=300 bytes) must call + update_quota with size=300 and add=False. + """ + folder = self.root_node.append_folder('big_folder') + v1 = factories.FileVersionFactory(size=100) + f1 = folder.append_file('x.txt'); f1.add_version(v1); f1.save() + v2 = factories.FileVersionFactory(size=200) + f2 = folder.append_file('y.txt'); f2.add_version(v2); f2.save() + + FileInfo.objects.update_or_create(file=f1, defaults={'file_size': 100}) + FileInfo.objects.update_or_create(file=f2, defaults={'file_size': 200}) + + resp = self._delete(folder) + assert resp.status_code == 200 + + assert mock_uq.called + call_args = mock_uq.call_args + size_arg = call_args[0][1] + add_arg = call_args[1].get('add', call_args[0][-1]) + assert size_arg == 300 + assert add_arg is False + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_delete_folder_zeroes_fileinfo_before_deletion(self, mock_st, mock_uq): + """ + The FileInfo records must be zeroed BEFORE the hard deletion of the + folder to prevent race conditions (views.py line 541). + """ + folder = self.root_node.append_folder('zeroed_folder') + v = factories.FileVersionFactory(size=500) + f = folder.append_file('z.txt'); f.add_version(v); f.save() + FileInfo.objects.update_or_create(file=f, defaults={'file_size': 500}) + + # We need to capture the FileInfo state at the moment update_quota is + # called – at that point file_size must already be 0. + recorded_file_size = [] + + def side_effect(target, size, storage_type, add): + fi = FileInfo.objects.filter(file=f).first() + if fi: + recorded_file_size.append(fi.file_size) + + mock_uq.side_effect = side_effect + + self._delete(folder) + + assert len(recorded_file_size) == 1 + assert recorded_file_size[0] == 0, ( + 'FileInfo.file_size must be 0 when update_quota is called, ' + f'but was {recorded_file_size[0]}' + ) + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_delete_empty_folder_no_quota_update(self, mock_st, mock_uq): + """ + Deleting an empty folder (total_size=0) must NOT call update_quota + (views.py line 538: if total_size > 0). + """ + folder = self.root_node.append_folder('empty_folder') + resp = self._delete(folder) + assert resp.status_code == 200 + assert not mock_uq.called + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_delete_folder_nested_files_quota(self, mock_st, mock_uq): + """ + Nested files (folder-in-folder) must ALL be counted when computing + the quota deduction. + """ + outer = self.root_node.append_folder('outer') + inner = outer.append_folder('inner') + + v1 = factories.FileVersionFactory(size=50) + f1 = inner.append_file('deep1.txt'); f1.add_version(v1); f1.save() + v2 = factories.FileVersionFactory(size=150) + f2 = inner.append_file('deep2.txt'); f2.add_version(v2); f2.save() + + FileInfo.objects.update_or_create(file=f1, defaults={'file_size': 50}) + FileInfo.objects.update_or_create(file=f2, defaults={'file_size': 150}) + + resp = self._delete(outer) + assert resp.status_code == 200 + + assert mock_uq.called + size_arg = mock_uq.call_args[0][1] + assert size_arg == 200 # 50 + 150 + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + def test_delete_file_does_not_trigger_folder_quota_path(self, mock_st, mock_uq): + """ + Deleting a single FILE must NOT go through the folder-quota path. + The new code block is guarded by `not file_node.is_file`. + """ + v = factories.FileVersionFactory(size=777) + f = self.root_node.append_file('single.txt') + f.add_version(v); f.save() + FileInfo.objects.update_or_create(file=f, defaults={'file_size': 777}) + + resp = self._delete(f) + assert resp.status_code == 200 + + # update_quota must NOT have been called by the folder-quota path. + # (The existing file-quota path in views.py does NOT call update_quota + # on delete; that is handled by the WaterButler hook elsewhere.) + assert not mock_uq.called + + +# --------------------------------------------------------------------------- +# Integration: full upload flow – osfstorage_create_child + create_waterbutler_log +# --------------------------------------------------------------------------- + +@pytest.mark.django_db +class TestUploadIntegration(StorageTestCase): + """ + Integration tests simulating the complete OSF upload flow. + + Step 1 – osfstorage_create_child: + WaterButler calls this endpoint after writing the file to storage. + Creates a FileInfo record and calls update_quota ONCE. + + Step 2 – create_waterbutler_log: + WaterButler calls this endpoint to record the log. + Fires file_signals.file_updated → update_used_quota. + The skip logic (provider == 'osfstorage') prevents update_quota + from being called a SECOND time (Critical 1 fix). + """ + + def setUp(self): + super().setUp() + self.root_node = self.node_settings.get_root() + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _upload(self, parent, name, size=123, unique_hash=None): + """POST to osfstorage_create_child (WaterButler upload callback).""" + payload = make_payload(user=self.user, name=name) + payload['metadata']['size'] = size + if unique_hash is not None: + # Different hash → genuinely new FileVersion (not treated as duplicate) + payload['metadata']['name'] = unique_hash + return self.app.post_json( + api_url_for( + 'osfstorage_create_child', + fid=parent._id, + guid=self.project._id, + ), + signing.sign_data(signing.default_signer, payload), + expect_errors=True, + ) + + def _wb_log(self, file_id, name, size, action='create'): + """PUT to create_waterbutler_log (WaterButler log callback).""" + options = { + 'auth': {'id': self.user._id}, + 'action': action, + 'provider': 'osfstorage', + 'metadata': { + 'provider': 'osfstorage', + 'name': name, + 'materialized': '/' + file_id, + 'path': '/' + file_id, + 'kind': 'file', + 'size': size, + 'created_utc': '', + 'modified_utc': '', + 'extra': {'version': '1'}, + }, + 'time': time_module.time() + 1000, + } + message, signature = signing.default_signer.sign_payload(options) + return self.app.put_json( + self.project.api_url_for('create_waterbutler_log'), + {'payload': message, 'signature': signature}, + headers={'Content-Type': 'application/json'}, + expect_errors=True, + ) + + # ------------------------------------------------------------------ + # Upload new file + # ------------------------------------------------------------------ + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.base.views.BaseFileNode') + @mock.patch('addons.base.views.timestamp') + def test_new_upload_wb_log_returns_200_and_no_duplicate( + self, mock_ts, mock_bfn, mock_st, mock_uq): + """ + Scenario #1: Upload a brand-new file. + + Expected: + - osfstorage_create_child: returns 200/201, creates FileInfo, calls update_quota once. + - create_waterbutler_log: returns 200 (not 500), creates a NodeLog entry. + - After both steps: still exactly 1 FileInfo record; update_quota NOT called again via signal. + """ + name = 'new_upload.txt' + size = 1024 + + # Step 1: WaterButler upload callback -> osfstorage_create_child + upload_resp = self._upload(self.root_node, name, size=size) + assert upload_resp.status_code in (200, 201), ( + f'osfstorage_create_child returned {upload_resp.status_code}' + ) + + # Scenario #4: exactly 1 FileInfo record created + record = self.root_node.find_child_by_name(name) + assert record is not None + assert FileInfo.objects.filter(file=record).count() == 1, ( + 'FileInfo must have exactly 1 record after upload.' + ) + + # Scenario #3: update_quota called exactly once from create_child + assert mock_uq.call_count == 1, ( + f'update_quota should be called once from osfstorage_create_child, ' + f'but was called {mock_uq.call_count} time(s).' + ) + mock_uq.reset_mock() + + # Step 2: WaterButler log callback -> create_waterbutler_log + nlogs = self.project.logs.count() + log_resp = self._wb_log(record._id, name, size) + + # Scenario #1: /create_waterbutler_log must return 200 + assert log_resp.status_code == 200, ( + f'create_waterbutler_log returned {log_resp.status_code} ' + '(expected 200). This was Critical 1: FILE_ADDED signal caused ' + 'duplicate FileInfo creation -> IntegrityError -> 500.' + ) + + # NodeLog must be created + self.project.reload() + assert self.project.logs.count() == nlogs + 1, ( + 'A NodeLog entry must be created by create_waterbutler_log.' + ) + + # Scenario #4: FileInfo still exactly 1 record (signal was skipped) + assert FileInfo.objects.filter(file=record).count() == 1, ( + 'FileInfo must remain exactly 1 record after create_waterbutler_log. ' + 'The skip logic in update_used_quota must prevent duplication.' + ) + + # Scenario #3: update_quota must NOT be called again via signal + assert mock_uq.call_count == 0, ( + f'update_quota was called {mock_uq.call_count} extra time(s) via ' + 'the FILE_ADDED signal. The skip logic should have prevented this.' + ) + + # ------------------------------------------------------------------ + # Overwrite: upload a new version of an existing file + # ------------------------------------------------------------------ + + @mock.patch('addons.osfstorage.views.update_quota') + @mock.patch('addons.osfstorage.views.get_project_storage_type', return_value=1) + @mock.patch('addons.base.views.BaseFileNode') + @mock.patch('addons.base.views.timestamp') + def test_overwrite_upload_wb_log_returns_200_and_no_duplicate( + self, mock_ts, mock_bfn, mock_st, mock_uq): + """ + Scenario #2: Upload a new version of an existing file (same filename). + + Expected: + - create_waterbutler_log returns 200. + - A NodeLog entry is created. + - FileInfo is UPDATED (UPSERT), not duplicated -> still exactly 1 record. + - update_quota called once from create_child for the size delta. + """ + name = 'overwrite.txt' + size_v1, size_v2 = 500, 800 + + # First upload + self._upload(self.root_node, name, size=size_v1, unique_hash='hash_v1') + record = self.root_node.find_child_by_name(name) + assert FileInfo.objects.filter(file=record).count() == 1 + mock_uq.reset_mock() + + # Second upload (same filename, different hash -> genuinely new FileVersion) + upload_resp = self._upload( + self.root_node, name, size=size_v2, unique_hash='hash_v2' + ) + assert upload_resp.status_code in (200, 201) + record.reload() + + # FileInfo must be UPDATED, not duplicated + fi = FileInfo.objects.filter(file=record).first() + assert fi is not None + assert fi.file_size == size_v2, ( + f'FileInfo.file_size should be {size_v2} after overwrite, got {fi.file_size}.' + ) + assert FileInfo.objects.filter(file=record).count() == 1 + + # update_quota called once for the new version (delta = size_v2 - size_v1) + assert mock_uq.call_count == 1 + mock_uq.reset_mock() + + # WaterButler log callback (action='update' for an overwrite) + nlogs = self.project.logs.count() + log_resp = self._wb_log(record._id, name, size_v2, action='update') + + # Scenario #2: /create_waterbutler_log must return 200 + assert log_resp.status_code == 200, ( + f'create_waterbutler_log returned {log_resp.status_code} ' + 'for overwrite (expected 200).' + ) + + self.project.reload() + assert self.project.logs.count() == nlogs + 1 + + # Still exactly 1 FileInfo record + assert FileInfo.objects.filter(file=record).count() == 1, ( + 'FileInfo must remain exactly 1 record after overwrite + ' + 'create_waterbutler_log.' + ) + + # update_quota must NOT be called again via signal (osfstorage skip) + assert mock_uq.call_count == 0, ( + f'update_quota was called {mock_uq.call_count} extra time(s) via ' + 'signal after overwrite. Skip logic should prevent this.' + ) + + @mock.patch('addons.base.views.BaseFileNode') + @mock.patch('addons.base.views.timestamp') + def test_upload_actual_quota_increment_no_double_count(self, mock_ts, mock_bfn): + """ + Do not mock update_quota — verify UserQuota.used in the database: + - After osfstorage_create_child: used increases exactly by file_size. + - After create_waterbutler_log: used does not increase further (no double-count). + """ + from osf.models import UserQuota + + # Ensure UserQuota initially exists with used=0 + uq, _ = UserQuota.objects.get_or_create( + user=self.user, + storage_type=UserQuota.NII_STORAGE, + defaults={'max_quota': 200, 'used': 0}, + ) + initial_used = UserQuota.objects.get( + user=self.user, storage_type=UserQuota.NII_STORAGE + ).used + + # Step 1: osfstorage_create_child + self._upload(self.root_node, 'real.txt', size=500) + record = self.root_node.find_child_by_name('real.txt') + assert record is not None + + after_create_child = UserQuota.objects.get( + user=self.user, storage_type=UserQuota.NII_STORAGE + ).used + assert after_create_child - initial_used == 500, \ + f'Initial increment incorrect: expected +500, got +{after_create_child - initial_used}' + + # Step 2: create_waterbutler_log + log_resp = self._wb_log(record._id, 'real.txt', 500) + assert log_resp.status_code == 200 + + final = UserQuota.objects.get( + user=self.user, storage_type=UserQuota.NII_STORAGE + ).used + assert final == after_create_child, \ + f'No double-counting after create_waterbutler_log: expected {after_create_child}, got {final}' diff --git a/addons/osfstorage/views.py b/addons/osfstorage/views.py index 1e413f84df2..3c3715ab636 100644 --- a/addons/osfstorage/views.py +++ b/addons/osfstorage/views.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from django.db.models import IntegerField +from django.db.models import IntegerField, Sum from django.db.models.functions import Cast from rest_framework import status as http_status import logging @@ -19,7 +19,7 @@ from api.caching.tasks import update_storage_usage from osf.exceptions import InvalidTagError, TagNotFoundError -from osf.models import FileVersion, OSFUser, ExportDataRestore, ExportData +from osf.models import FileVersion, OSFUser, ExportDataRestore, ExportData, FileInfo, BaseFileNode from osf.utils.permissions import WRITE from osf.utils.requests import check_select_for_update from website.project.decorators import ( @@ -28,6 +28,7 @@ from website.project.model import has_anonymous_link from website.files import exceptions +from website.util.quota import get_project_storage_type, update_quota from addons.osfstorage import utils from addons.osfstorage import decorators from addons.osfstorage.models import OsfStorageFolder @@ -124,17 +125,110 @@ def osfstorage_get_revisions(file_node, payload, target, **kwargs): ] } +def _get_all_descendant_file_ids(folder_node): + TRASHED_TYPES = ('osf.trashedfilenode', 'osf.trashedfile', 'osf.trashedfolder') + + file_ids = [] + folder_ids_to_process = [folder_node.id] + + while folder_ids_to_process: + children = BaseFileNode.objects.filter( + parent_id__in=folder_ids_to_process, + provider='osfstorage', + ).exclude( + type__in=TRASHED_TYPES + ).values('id', 'type') + + next_folder_ids = [] + for child in children: + if child['type'] == 'osf.osfstoragefile': + file_ids.append(child['id']) + elif child['type'] == 'osf.osfstoragefolder': + next_folder_ids.append(child['id']) + + folder_ids_to_process = next_folder_ids + + return file_ids @decorators.waterbutler_opt_hook def osfstorage_copy_hook(source, destination, name=None, **kwargs): - ret = source.copy_under(destination, name=name).serialize(), http_status.HTTP_201_CREATED + replaced_size = int(kwargs.get('replaced_size', 0)) + cloned = source.copy_under(destination, name=name) + + # Only handle if source is a node and destination is a node + if getattr(source.target, 'type', '') == 'osf.node' and getattr(destination.target, 'type', '') == 'osf.node': + if source.is_file: + latest_version = cloned.versions.order_by('-created').first() + if latest_version and latest_version.size is not None and latest_version.size > 0: + new_size = latest_version.size + else: + if latest_version and (latest_version.size is None or latest_version.size < 0): + logger.warning( + 'latest_version.size is %r for cloned file %s (osfstorage_copy_hook); ' + 'treating size as 0 for quota calculation', latest_version.size, cloned._id + ) + new_size = 0 + FileInfo.objects.update_or_create(file=cloned, defaults={'file_size': new_size}) + else: + source_file_ids = _get_all_descendant_file_ids(source) + source_size_map = { + fi.file_id: fi.file_size + for fi in FileInfo.objects.filter(file_id__in=source_file_ids) + } + + cloned_file_ids = _get_all_descendant_file_ids(cloned) + cloned_files = list(BaseFileNode.objects.filter( + id__in=cloned_file_ids + ).values('id', 'copied_from_id')) + + file_infos = [] + new_size = 0 + for cf in cloned_files: + src_id = cf['copied_from_id'] + if src_id in source_size_map: + size = source_size_map[src_id] + elif src_id is not None: + # FileInfo not exist + node = BaseFileNode.objects.filter(id=src_id).first() + size = 0 + if node: + v = node.versions.order_by('-created').first() + if v and v.size is not None and v.size > 0: + size = v.size + else: + if v and (v.size is None or v.size < 0): + logger.warning( + 'v.size is %r for source file %s (osfstorage_copy_hook FOLDER branch); ' + 'treating size as 0 for quota calculation', v.size, src_id + ) + size = 0 + else: # src_id is None and not in map + size = 0 + + file_infos.append(FileInfo(file_id=cf['id'], file_size=size)) + new_size += size + + try: + FileInfo.objects.bulk_create(file_infos) + except IntegrityError: + for fi in file_infos: + FileInfo.objects.get_or_create(file_id=fi.file_id, defaults={'file_size': fi.file_size}) + replaced_size = 0 # osfstorage_delete already subtracted this + + # UserQuota: delta = new_size - replaced_size + delta = new_size - replaced_size + if delta != 0: + storage_type = get_project_storage_type(destination.target) + update_quota(destination.target, abs(delta), storage_type, add=(delta > 0)) + update_storage_usage(destination.target) - return ret + return cloned.serialize(), http_status.HTTP_201_CREATED @decorators.waterbutler_opt_hook def osfstorage_move_hook(source, destination, name=None, **kwargs): source_target = source.target is_check_permission = kwargs.get('is_check_permission') + replaced_size = int(kwargs.get('replaced_size', 0)) try: ret = source.move_under(destination, name=name, is_check_permission=is_check_permission).serialize(), http_status.HTTP_200_OK except exceptions.FileNodeCheckedOutError: @@ -150,6 +244,57 @@ def osfstorage_move_hook(source, destination, name=None, **kwargs): 'message_long': 'Cannot move file as it is the primary file of preprint.' }) + # Only handle if source is a node and destination is a node + if getattr(source_target, 'type', '') == 'osf.node' and getattr(destination.target, 'type', '') == 'osf.node': + if source.is_file: + latest_version = source.versions.order_by('-created').first() + if latest_version and latest_version.size is not None and latest_version.size > 0: + new_size = latest_version.size + else: + if latest_version and (latest_version.size is None or latest_version.size < 0): + logger.warning( + 'latest_version.size is %r for moved file %s (osfstorage_move_hook); ' + 'treating size as 0 for quota calculation', latest_version.size, source._id + ) + new_size = 0 + # FileInfo: UPSERT + FileInfo.objects.update_or_create(file=source, defaults={'file_size': new_size}) + else: + source_file_ids = _get_all_descendant_file_ids(source) # recursive + new_size = FileInfo.objects.filter( + file_id__in=source_file_ids + ).aggregate(total=Sum('file_size'))['total'] or 0 + replaced_size = 0 # osfstorage_delete already subtracted this + + # UserQuota + src_storage = get_project_storage_type(source_target) + dest_storage = get_project_storage_type(destination.target) + + # "Same UserQuota record" = same (creator, storage_type), matching the criterion + # WaterButler's resolve_quota_context() uses for the pre-check (user_guid + + # storage_type). + # Node identity is NOT the right test: a move to another component owned by the + # same creator lands on the very same UserQuota row, so subtracting from the + # source and adding to the destination would be two writes against one row -- + # and update_quota() floors `used` at 0 between them, which permanently corrupts + # the row whenever used < new_size. + same_user_quota = ( + source_target.creator_id == destination.target.creator_id and + src_storage == dest_storage + ) + + if same_user_quota: + # Net delta is -replaced_size: the moved bytes are already counted in this + # very row, only the overwritten bytes are freed. (For folders replaced_size + # was reset to 0 above -- osfstorage_delete already subtracted them.) + if replaced_size > 0: + update_quota(destination.target, replaced_size, dest_storage, add=False) + else: + update_quota(source_target, new_size, src_storage, add=False) + delta = new_size - replaced_size + if delta != 0: + update_quota(destination.target, abs(delta), dest_storage, add=(delta > 0)) + # once the move is complete recalculate storage for both targets if it's a inter-target move. if is_check_permission and source_target != destination.target: update_storage_usage(destination.target) @@ -379,6 +524,32 @@ def osfstorage_create_child(file_node, payload, **kwargs): if not current_version or not current_version.is_duplicate(new_version): update_storage_usage(file_node.target) + # Only handle if target is a node + if getattr(file_node.target, 'type', '') == 'osf.node': + if new_version.size is not None and new_version.size > 0: + new_size = new_version.size + else: + if new_version.size is None or new_version.size < 0: + logger.warning( + 'new_version.size is %r for created file %s (osfstorage_create_child); ' + 'treating size as 0 for quota calculation', new_version.size, file_node._id + ) + new_size = 0 + + old_info = FileInfo.objects.filter(file=file_node).first() + old_size = old_info.file_size if old_info else 0 + + # FileInfo: UPSERT + FileInfo.objects.update_or_create( + file=file_node, defaults={'file_size': new_size} + ) + + # UserQuota: delta = new - old + delta = new_size - old_size + if delta != 0: + storage_type = get_project_storage_type(file_node.target) + update_quota(file_node.target, abs(delta), storage_type, add=(delta > 0)) + version_id = new_version._id archive_exists = new_version.archive is not None else: @@ -406,6 +577,17 @@ def osfstorage_delete(file_node, payload, target, **kwargs): if file_node == OsfStorageFolder.objects.get_root(target=target): raise HTTPError(http_status.HTTP_400_BAD_REQUEST) + if getattr(file_node.target, 'type', '') == 'osf.node' and not file_node.is_file: + # Since we are deleting a folder, we need to calculate the total size of all descendant files to update quota + all_file_ids = _get_all_descendant_file_ids(file_node) + total_size = FileInfo.objects.filter(file_id__in=all_file_ids) \ + .aggregate(total=Sum('file_size'))['total'] or 0 + if total_size > 0: + storage_type = get_project_storage_type(target) + # Subtract quota BEFORE deletion; also zero FileInfo to avoid race conditions with new file creation during deletion + FileInfo.objects.filter(file_id__in=all_file_ids).update(file_size=0) + update_quota(target, total_size, storage_type, add=False) + try: file_node.delete(user=user) diff --git a/addons/s3/README.md b/addons/s3/README.md index 6b4261bd77a..d3347506daa 100644 --- a/addons/s3/README.md +++ b/addons/s3/README.md @@ -16,3 +16,23 @@ If you already have an access key and ID, skip this step 2. Scroll down to Configure Add-ons 3. Connect your account and enter your ID and secret 4. Select a bucket to work from, or create a new one. + + +## Creating a restricted-access AWS user for S3 connection + +1. Login to AWS +2. Open Identity and Access Management +3. Create a user, assign name, set "Access Type" to "Programmatic Access" +4. Permissions (simple): + 1. Add "AmazonS3FullAccess" policy +5. Permissions (minimal): + 1. Click "Create Policy" => opens a new window + 2. Make a new policy with the following Actions enabled: `["s3:DeleteObject", "s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:ReplicateObject", "s3:RestoreObject", "s3:ListAllMyBuckets", "s3:GetBucketLocation", "s3:CreateBucket"]` + 3. Resources: *I just do "All Resources", but an AWS-educated person would know better about how to narrow it down. + 4. Click "Create" + 5. Return to "Create User" window + 6. Reload policy list + 7. Search for your new policy and select. +6. Tags: *eh* +7. Click "Review", review, then click "Create user" +8. Note "Access key ID" and "Secret access key". These are the inputs to the OSF S3 addon. \ No newline at end of file diff --git a/addons/s3/apps.py b/addons/s3/apps.py index 48943b4d87d..ef520daeed1 100644 --- a/addons/s3/apps.py +++ b/addons/s3/apps.py @@ -12,6 +12,7 @@ class S3AddonAppConfig(BaseAddonAppConfig): + default = True name = 'addons.s3' label = 'addons_s3' full_name = 'Amazon S3' diff --git a/addons/s3/models.py b/addons/s3/models.py index b42a1552696..226467776bb 100644 --- a/addons/s3/models.py +++ b/addons/s3/models.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from addons.base.models import (BaseOAuthNodeSettings, BaseOAuthUserSettings, BaseStorageAddon) from django.db import models @@ -8,11 +6,18 @@ from addons.base import exceptions from addons.s3.provider import S3Provider from addons.s3.serializer import S3Serializer -from addons.s3.settings import (BUCKET_LOCATIONS, - ENCRYPT_UPLOADS_DEFAULT) -from addons.s3.utils import (bucket_exists, - get_bucket_location_or_error, - get_bucket_names) +from addons.s3.settings import ( + BUCKET_LOCATIONS, + ENCRYPT_UPLOADS_DEFAULT +) +from addons.s3.utils import ( + bucket_exists, + get_bucket_location_or_error, + get_bucket_names, + get_bucket_prefixes +) +from website.util import api_v2_url + class S3FileNode(BaseFileNode): _provider = 's3' @@ -53,10 +58,12 @@ def folder_path(self): @property def display_name(self): - return u'{0}: {1}'.format(self.config.full_name, self.folder_id) + return f'{self.config.full_name}: {self.folder_id}' def set_folder(self, folder_id, auth): - if not bucket_exists(self.external_account.oauth_key, self.external_account.oauth_secret, folder_id): + bucket_name = folder_id.split(':')[0] + + if not bucket_exists(self.external_account.oauth_key, self.external_account.oauth_secret, bucket_name): error_message = ('We are having trouble connecting to that bucket. ' 'Try a different one.') raise exceptions.InvalidFolderError(error_message) @@ -66,7 +73,7 @@ def set_folder(self, folder_id, auth): bucket_location = get_bucket_location_or_error( self.external_account.oauth_key, self.external_account.oauth_secret, - folder_id + bucket_name ) try: bucket_location = BUCKET_LOCATIONS[bucket_location] @@ -75,32 +82,46 @@ def set_folder(self, folder_id, auth): # Default to the key. When hit, add mapping to settings pass - self.folder_name = '{} ({})'.format(folder_id, bucket_location) + self.folder_name = f'{folder_id} ({bucket_location})' self.save() - self.nodelogger.log(action='bucket_linked', extra={'bucket': str(folder_id)}, save=True) + self.nodelogger.log(action='bucket_linked', extra={'bucket': bucket_name, 'path': self.folder_id}, save=True) - def get_folders(self, **kwargs): - # This really gets only buckets, not subfolders, - # as that's all we want to be linkable on a node. - try: + def get_folders(self, path, folder_id): + """ + Our S3 implementation allows for folder_id to be a bucket's root, or a subfolder in that bucket. + """ + # This is the root, so list all buckets. + if not folder_id: buckets = get_bucket_names(self) - except Exception: - raise exceptions.InvalidAuthError() - return [ - { + return [{ 'addon': 's3', 'kind': 'folder', - 'id': bucket, + 'id': f'{bucket}:/', 'name': bucket, - 'path': bucket, + 'bucket_name': bucket, + 'path': '/', 'urls': { - 'folders': '' + 'folders': api_v2_url( + f'nodes/{self.owner._id}/addons/s3/folders/', + params={ + 'id': bucket, + 'bucket_name': bucket, + } + ), } - } - for bucket in buckets - ] + } for bucket in buckets] + # This is for a directory for a specific bucket, folders (Prefixes), but not files (Keys) returned, because + # these we can only set folders as our base folder_id + else: + bucket_name, _, path = folder_id.partition(':/') + return get_bucket_prefixes( + self.external_account.oauth_key, + self.external_account.oauth_secret, + prefix=path, + bucket_name=bucket_name + ) @property def complete(self): @@ -124,7 +145,7 @@ def deauthorize(self, auth=None, log=True): def delete(self, save=True): self.deauthorize(log=False) - super(NodeSettings, self).delete(save=save) + super().delete(save=save) def serialize_waterbutler_credentials(self): if not self.has_auth: @@ -135,10 +156,16 @@ def serialize_waterbutler_credentials(self): } def serialize_waterbutler_settings(self): + """ + We use the folder id to hold the bucket location + """ if not self.folder_id: raise exceptions.AddonError('Cannot serialize settings for S3 addon') + + bucket_name = self.folder_id.split(':')[0] return { - 'bucket': self.folder_id, + 'bucket': bucket_name, + 'id': self.folder_id, 'encrypt_uploads': self.encrypt_uploads } @@ -146,7 +173,7 @@ def create_waterbutler_log(self, auth, action, metadata): url = self.owner.web_url_for('addon_view_or_download_file', path=metadata['path'], provider='s3') self.owner.add_log( - 's3_{0}'.format(action), + f's3_{action}', auth=auth, params={ 'project': self.owner.parent_id, diff --git a/addons/s3/provider.py b/addons/s3/provider.py index ba28d065d54..0321a4a7d4e 100644 --- a/addons/s3/provider.py +++ b/addons/s3/provider.py @@ -1,6 +1,6 @@ from addons.s3.serializer import S3Serializer -class S3Provider(object): +class S3Provider: """An alternative to `ExternalProvider` not tied to OAuth""" name = 'Amazon S3' @@ -8,7 +8,7 @@ class S3Provider(object): serializer = S3Serializer def __init__(self, account=None): - super(S3Provider, self).__init__() + super().__init__() # provide an unauthenticated session by default self.account = account diff --git a/addons/s3/requirements.txt b/addons/s3/requirements.txt index 8e6ed87adf9..a61cef6d77f 100644 --- a/addons/s3/requirements.txt +++ b/addons/s3/requirements.txt @@ -1 +1,2 @@ -boto==2.38.0 +# boto (v2) was removed as part of SigV4 migration. +# boto3/botocore are declared in the root requirements.txt. diff --git a/addons/s3/settings/__init__.py b/addons/s3/settings/__init__.py index 2b2f98881f6..40f955c5a78 100644 --- a/addons/s3/settings/__init__.py +++ b/addons/s3/settings/__init__.py @@ -7,4 +7,4 @@ try: from .local import * # noqa except ImportError: - logger.warn('No local.py settings file found') + logger.warning('No local.py settings file found') diff --git a/addons/s3/static/node-cfg.js b/addons/s3/static/node-cfg.js index c7e33ca410a..ccef5f3e251 100644 --- a/addons/s3/static/node-cfg.js +++ b/addons/s3/static/node-cfg.js @@ -4,4 +4,4 @@ var s3NodeConfig = require('./s3NodeConfig.js').s3NodeConfig; var url = window.contextVars.node.urls.api + 's3/settings/'; -new s3NodeConfig('Amazon S3', '#s3Scope', url, '#s3Grid'); +new s3NodeConfig('Amazon S3', '#s3Scope', url, '#s3Grid'); \ No newline at end of file diff --git a/addons/s3/static/s3-rubeus-cfg.js b/addons/s3/static/s3-rubeus-cfg.js index 8493351f9ee..d76dcc8f744 100644 --- a/addons/s3/static/s3-rubeus-cfg.js +++ b/addons/s3/static/s3-rubeus-cfg.js @@ -1,47 +1,46 @@ - var Rubeus = require('rubeus'); +var Rubeus = require('rubeus'); - Rubeus.cfg.s3 = { +Rubeus.cfg.s3 = { - uploadMethod: 'PUT', - uploadUrl: null, - uploadAdded: function(file, item) { - var self = this; - var parent = self.getByID(item.parentID); - var name = file.name; - // Make it possible to upload into subfolders - while (parent.depth > 1 && !parent.isAddonRoot) { - name = parent.name + '/' + name; - parent = self.getByID(parent.parentID); - } - file.destination = name; - file.signedUrlFrom = parent.urls.upload; - }, + uploadMethod: 'PUT', + uploadUrl: null, + uploadAdded: function (file, item) { + var self = this; + var parent = self.getByID(item.parentID); + var name = file.name; + // Make it possible to upload into subfolders + while (parent.depth > 1 && !parent.isAddonRoot) { + name = parent.name + '/' + name; + parent = self.getByID(parent.parentID); + } + file.destination = name; + file.signedUrlFrom = parent.urls.upload; + }, - uploadSending: function(file, formData, xhr) { - xhr.setRequestHeader('Content-Type', file.type || 'application/octet-stream'); - xhr.setRequestHeader('x-amz-acl', 'private'); - }, + uploadSending: function (file, formData, xhr) { + xhr.setRequestHeader('Content-Type', file.type || 'application/octet-stream'); + xhr.setRequestHeader('x-amz-acl', 'private'); + }, - uploadSuccess: function(file, row) { - var self = this; - var parent = this.getByID(row.parentID); - row.urls = { - 'delete': parent.nodeApiUrl + 's3/' + file.destination + '/', - 'download': parent.nodeUrl + 's3/' + file.destination + '/download/', - 'view': parent.nodeUrl + 's3/' + file.destination + '/' - }; - row.permissions = parent.permissions; - this.updateItem(row); - var updated = Rubeus.Utils.itemUpdated(row, parent); - if (updated) { - self.changeStatus(row, Rubeus.Status.UPDATED); - self.delayRemoveRow(row); - } else { - self.changeStatus(row, Rubeus.Status.UPLOAD_SUCCESS, null, 2000, - function(row) { - self.showButtons(row); - }); - } + uploadSuccess: function (file, row) { + var self = this; + var parent = this.getByID(row.parentID); + row.urls = { + 'delete': parent.nodeApiUrl + 's3/' + file.destination + '/', + 'download': parent.nodeUrl + 's3/' + file.destination + '/download/', + 'view': parent.nodeUrl + 's3/' + file.destination + '/' + }; + row.permissions = parent.permissions; + this.updateItem(row); + var updated = Rubeus.Utils.itemUpdated(row, parent); + if (updated) { + self.changeStatus(row, Rubeus.Status.UPDATED); + self.delayRemoveRow(row); + } else { + self.changeStatus(row, Rubeus.Status.UPLOAD_SUCCESS, null, 2000, + function (row) { + self.showButtons(row); + }); } - }; - + } +}; diff --git a/addons/s3/static/s3AnonymousLogActionList.json b/addons/s3/static/s3AnonymousLogActionList.json index 788ddcfa5fd..9f470e89f03 100644 --- a/addons/s3/static/s3AnonymousLogActionList.json +++ b/addons/s3/static/s3AnonymousLogActionList.json @@ -1,11 +1,11 @@ { - "s3_bucket_linked" : "A user linked an Amazon S3 bucket to a project", - "s3_bucket_unlinked" : "A user unselected an Amazon S3 bucket in a project", - "s3_file_added" : "A user added a file to an Amazon S3 bucket in a project", - "s3_file_removed" : "A user removed a file in an Amazon S3 bucket in a project", - "s3_file_updated" : "A user updated a file in an Amazon S3 bucket in a project", - "s3_folder_created" : "A user created a folder in an Amazon S3 in a project", - "s3_node_authorized" : "A user authorized the Amazon S3 addon for a project", - "s3_node_deauthorized" : "A user deauthorized the Amazon S3 addon for a project", - "s3_node_deauthorized_no_user" : "Amazon S3 addon for a project deauthorized" -} + "s3_bucket_linked": "A user linked an Amazon S3 bucket to a project", + "s3_bucket_unlinked": "A user unselected an Amazon S3 bucket in a project", + "s3_file_added": "A user added a file to an Amazon S3 bucket in a project", + "s3_file_removed": "A user removed a file in an Amazon S3 bucket in a project", + "s3_file_updated": "A user updated a file in an Amazon S3 bucket in a project", + "s3_folder_created": "A user created a folder in an Amazon S3 in a project", + "s3_node_authorized": "A user authorized the Amazon S3 addon for a project", + "s3_node_deauthorized": "A user deauthorized the Amazon S3 addon for a project", + "s3_node_deauthorized_no_user": "Amazon S3 addon for a project deauthorized" +} \ No newline at end of file diff --git a/addons/s3/static/s3LogActionList.json b/addons/s3/static/s3LogActionList.json index f2dc7ceef05..63d88ca3ba7 100644 --- a/addons/s3/static/s3LogActionList.json +++ b/addons/s3/static/s3LogActionList.json @@ -1,11 +1,11 @@ { - "s3_bucket_linked" : "${user} linked the Amazon S3 bucket ${bucket} to ${node}", - "s3_bucket_unlinked" : "${user} unselected the Amazon S3 bucket ${bucket} in ${node}", - "s3_file_added" : "${user} added file ${path} to Amazon S3 bucket ${bucket} in ${node}", - "s3_file_removed" : "${user} removed ${path} in Amazon S3 bucket ${bucket} in ${node}", - "s3_file_updated" : "${user} updated file ${path} in Amazon S3 bucket ${bucket} in ${node}", - "s3_folder_created" : "${user} created folder ${path} in Amazon S3 bucket ${bucket} in ${node}", - "s3_node_authorized" : "${user} authorized the Amazon S3 addon for ${node}", - "s3_node_deauthorized" : "${user} deauthorized the Amazon S3 addon for ${node}", - "s3_node_deauthorized_no_user" : "Amazon S3 addon for ${node} deauthorized" -} + "s3_bucket_linked": "${user} linked the Amazon S3 bucket ${bucket} to ${node}", + "s3_bucket_unlinked": "${user} unselected the Amazon S3 bucket ${bucket} in ${node}", + "s3_file_added": "${user} added file ${path} to Amazon S3 bucket ${bucket} in ${node}", + "s3_file_removed": "${user} removed ${path} in Amazon S3 bucket ${bucket} in ${node}", + "s3_file_updated": "${user} updated file ${path} in Amazon S3 bucket ${bucket} in ${node}", + "s3_folder_created": "${user} created folder ${path} in Amazon S3 bucket ${bucket} in ${node}", + "s3_node_authorized": "${user} authorized the Amazon S3 addon for ${node}", + "s3_node_deauthorized": "${user} deauthorized the Amazon S3 addon for ${node}", + "s3_node_deauthorized_no_user": "Amazon S3 addon for ${node} deauthorized" +} \ No newline at end of file diff --git a/addons/s3/static/s3NodeConfig.js b/addons/s3/static/s3NodeConfig.js index f5369691dfb..071deaed584 100644 --- a/addons/s3/static/s3NodeConfig.js +++ b/addons/s3/static/s3NodeConfig.js @@ -15,7 +15,7 @@ var OauthAddonFolderPicker = require('js/oauthAddonNodeConfig')._OauthAddonNodeC var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { bucketLocations: s3Settings.bucketLocations, - constructor: function(addonName, url, selector, folderPicker, opts, tbOpts) { + constructor: function (addonName, url, selector, folderPicker, opts, tbOpts) { var self = this; self.super.constructor(addonName, url, selector, folderPicker, tbOpts); // Non-OAuth fields @@ -26,9 +26,9 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { {}, OauthAddonFolderPicker.prototype.treebeardOptions, { // TreeBeard Options - columnTitles: function() { + columnTitles: function () { return [{ - title: 'Buckets', + title: 'Buckets/Folders', width: '75%', sort: false }, { @@ -37,30 +37,24 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { sort: false }]; }, - resolveToggle: function(item) { - return ''; - }, - resolveIcon: function(item) { - return m('i.fa.fa-folder-o', ' '); - }, }, tbOpts ); }, - connectAccount: function() { + connectAccount: function () { var self = this; - if( !self.accessKey() && !self.secretKey() ){ + if (!self.accessKey() && !self.secretKey()) { self.changeMessage('Please enter both an API access key and secret key.', 'text-danger'); return; } - if (!self.accessKey() ){ + if (!self.accessKey()) { self.changeMessage('Please enter an API access key.', 'text-danger'); return; } - if (!self.secretKey() ){ + if (!self.secretKey()) { self.changeMessage('Please enter an API secret key.', 'text-danger'); return; } @@ -68,17 +62,17 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { return $osf.postJSON( self.urls().create, { - secret_key: self.secretKey(), - access_key: self.accessKey() - } - ).done(function(response) { + secret_key: self.secretKey(), + access_key: self.accessKey() + } + ).done(function (response) { $osf.unblock(); self.clearModal(); $('#s3InputCredentials').modal('hide'); self.changeMessage('Successfully added S3 credentials.', 'text-success', null, true); self.updateFromData(response); self.importAuth(); - }).fail(function(xhr, status, error) { + }).fail(function (xhr, status, error) { $osf.unblock(); var message = ''; var response = JSON.parse(xhr.responseText); @@ -110,7 +104,7 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { * @param {String} bucketName user-provided name of bucket to validate * @param {Boolean} laxChecking whether to use the more permissive validation */ - isValidBucketName: function(bucketName, laxChecking) { + isValidBucketName: function (bucketName, laxChecking) { if (laxChecking === true) { return /^[a-zA-Z0-9.\-_]{1,255}$/.test(bucketName); } @@ -122,7 +116,7 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { }, /** Reset all fields from S3 credentials input modal */ - clearModal: function() { + clearModal: function () { var self = this; self.message(''); self.messageClass('text-info'); @@ -130,22 +124,22 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { self.accessKey(null); }, - createBucket: function(self, bucketName, bucketLocation) { + createBucket: function (self, bucketName, bucketLocation) { $osf.block(); bucketName = bucketName.toLowerCase(); return $osf.postJSON( self.urls().createBucket, { - bucket_name: bucketName, - bucket_location: bucketLocation - } - ).done(function(response) { + bucket_name: bucketName, + bucket_location: bucketLocation + } + ).done(function (response) { $osf.unblock(); self.loadedFolders(false); self.activatePicker(); var msg = 'Successfully created bucket "' + $osf.htmlEscape(bucketName) + '". You can now select it from the list.'; var msgType = 'text-success'; self.changeMessage(msg, msgType, null, true); - }).fail(function(xhr) { + }).fail(function (xhr) { var resp = JSON.parse(xhr.responseText); var message = resp.message; var title = resp.title || 'Problem creating bucket'; @@ -156,21 +150,21 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { bootbox.confirm({ title: $osf.htmlEscape(title), message: $osf.htmlEscape(message), - callback: function(result) { + callback: function (result) { if (result) { self.openCreateBucket(); } }, - buttons:{ - confirm:{ - label:'Try again' + buttons: { + confirm: { + label: 'Try again' } } }); }); }, - openCreateBucket: function() { + openCreateBucket: function () { var self = this; // Generates html options for key-value pairs in BUCKET_LOCATION_MAP @@ -187,32 +181,32 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { bootbox.dialog({ title: 'Create a new bucket', message: - '
' + - '
' + - '
' + - '
' + - ' ' + - '
' + - ' ' + - '
' + - '' + - '
'+ - '
' + - '
' + - '
' + - ' ' + - '
' + - '' + - '
' + - '
' + - '
' + - 'For more information on locations, click ' + - 'here' + - '' + - '
' + - '
', + '
' + + '
' + + '
' + + '
' + + ' ' + + '
' + + ' ' + + '
' + + '' + + '
' + + '
' + + '
' + + '
' + + ' ' + + '
' + + '' + + '
' + + '
' + + '
' + + 'See more ' + + 'information on locations' + + '' + + '
' + + '
', buttons: { cancel: { label: 'Cancel', @@ -234,7 +228,7 @@ var s3FolderPickerViewModel = oop.extend(OauthAddonFolderPicker, { bootbox.confirm({ title: 'Invalid bucket name', message: 'Amazon S3 buckets can contain lowercase letters, numbers, and hyphens separated by' + - ' periods. Please try another name.', + ' periods. Please try another name.', callback: function (result) { if (result) { self.openCreateBucket(); @@ -271,4 +265,4 @@ function s3NodeConfig(addonName, selector, url, folderPicker, opts, tbOpts) { module.exports = { s3NodeConfig: s3NodeConfig, _s3NodeConfigViewModel: s3FolderPickerViewModel -}; +}; \ No newline at end of file diff --git a/addons/s3/static/s3UserConfig.js b/addons/s3/static/s3UserConfig.js index 5489bc93ca8..ed9c4d433b2 100644 --- a/addons/s3/static/s3UserConfig.js +++ b/addons/s3/static/s3UserConfig.js @@ -32,26 +32,26 @@ function ViewModel(url) { ChangeMessageMixin.call(self); /** Reset all fields from S3 credentials input modal */ - self.clearModal = function() { + self.clearModal = function () { self.message(''); self.messageClass('text-info'); self.accessKey(null); self.secretKey(null); }; /** Send POST request to authorize S3 */ - self.connectAccount = function() { + self.connectAccount = function () { // Selection should not be empty - if( !self.accessKey() && !self.secretKey() ){ + if (!self.accessKey() && !self.secretKey()) { self.changeMessage('Please enter both an API access key and secret key.', 'text-danger'); return; } - if (!self.accessKey() ){ + if (!self.accessKey()) { self.changeMessage('Please enter an API access key.', 'text-danger'); return; } - if (!self.secretKey() ){ + if (!self.secretKey()) { self.changeMessage('Please enter an API secret key.', 'text-danger'); return; } @@ -61,12 +61,12 @@ function ViewModel(url) { access_key: self.accessKey, secret_key: self.secretKey, }) - ).done(function() { + ).done(function () { self.clearModal(); $modal.modal('hide'); self.updateAccounts(); - }).fail(function(xhr, textStatus, error) { + }).fail(function (xhr, textStatus, error) { var errorMessage = (xhr.status === 400 && xhr.responseJSON.message !== undefined) ? xhr.responseJSON.message : language.authError; self.changeMessage(errorMessage, 'text-danger'); Raven.captureMessage('Could not authenticate with S3', { @@ -79,20 +79,20 @@ function ViewModel(url) { }); }; - self.updateAccounts = function() { + self.updateAccounts = function () { return $.ajax({ url: url, type: 'GET', dataType: 'json' }).done(function (data) { - self.accounts($.map(data.accounts, function(account) { - var externalAccount = new ExternalAccount(account); + self.accounts($.map(data.accounts, function (account) { + var externalAccount = new ExternalAccount(account); externalAccount.accessKey = account.oauth_key; externalAccount.secretKey = account.oauth_secret; return externalAccount; })); - $('#s3-header').osfToggleHeight({height: 160}); - }).fail(function(xhr, status, error) { + $('#s3-header').osfToggleHeight({ height: 160 }); + }).fail(function (xhr, status, error) { self.changeMessage(language.userSettingsError, 'text-danger'); Raven.captureMessage('Error while updating addon account', { extra: { @@ -104,7 +104,7 @@ function ViewModel(url) { }); }; - self.askDisconnect = function(account) { + self.askDisconnect = function (account) { var self = this; bootbox.confirm({ title: 'Disconnect Amazon S3 Account?', @@ -117,26 +117,26 @@ function ViewModel(url) { self.disconnectAccount(account); } }, - buttons:{ - confirm:{ - label:'Disconnect', - className:'btn-danger' + buttons: { + confirm: { + label: 'Disconnect', + className: 'btn-danger' } } }); }; - self.disconnectAccount = function(account) { + self.disconnectAccount = function (account) { var self = this; var url = '/api/v1/oauth/accounts/' + account.id + '/'; var request = $.ajax({ url: url, type: 'DELETE' }); - request.done(function(data) { + request.done(function (data) { self.updateAccounts(); }); - request.fail(function(xhr, status, error) { + request.fail(function (xhr, status, error) { Raven.captureMessage('Error while removing addon authorization for ' + account.id, { extra: { url: url, @@ -148,8 +148,8 @@ function ViewModel(url) { return request; }; - self.selectionChanged = function() { - self.changeMessage('',''); + self.selectionChanged = function () { + self.changeMessage('', ''); }; self.updateAccounts(); @@ -170,4 +170,4 @@ function S3UserConfig(selector, url) { module.exports = { S3ViewModel: ViewModel, S3UserConfig: S3UserConfig -}; +}; \ No newline at end of file diff --git a/addons/s3/static/settings.json b/addons/s3/static/settings.json index 05ea9dc29c5..1a5060a1ae4 100644 --- a/addons/s3/static/settings.json +++ b/addons/s3/static/settings.json @@ -16,4 +16,4 @@ "sa-east-1": "Sao Paulo" }, "encryptUploads": true -} +} \ No newline at end of file diff --git a/addons/s3/static/user-cfg.js b/addons/s3/static/user-cfg.js index 48e6c1894e5..474aa68ca73 100644 --- a/addons/s3/static/user-cfg.js +++ b/addons/s3/static/user-cfg.js @@ -3,4 +3,4 @@ var S3UserConfig = require('./s3UserConfig.js').S3UserConfig; // Endpoint for S3 user settings var url = '/api/v1/settings/s3/accounts/'; -var s3UserConfig = new S3UserConfig('#s3AddonScope', url); +var s3UserConfig = new S3UserConfig('#s3AddonScope', url); \ No newline at end of file diff --git a/addons/s3/templates/s3_credentials_modal.mako b/addons/s3/templates/s3_credentials_modal.mako index de0092a9035..f097291f03a 100644 --- a/addons/s3/templates/s3_credentials_modal.mako +++ b/addons/s3/templates/s3_credentials_modal.mako @@ -44,4 +44,4 @@ - + \ No newline at end of file diff --git a/addons/s3/templates/s3_node_settings.mako b/addons/s3/templates/s3_node_settings.mako index 809d86dbb2e..82a127a4e7c 100644 --- a/addons/s3/templates/s3_node_settings.mako +++ b/addons/s3/templates/s3_node_settings.mako @@ -4,7 +4,7 @@ <%include file="s3_credentials_modal.mako"/>

- + ${addon_full_name} icon ${addon_full_name} @@ -29,7 +29,7 @@

- + ${_("Connect Account")} @@ -90,4 +90,4 @@

- + \ No newline at end of file diff --git a/addons/s3/templates/s3_user_settings.mako b/addons/s3/templates/s3_user_settings.mako index dc4274d9319..83f7317d1ab 100644 --- a/addons/s3/templates/s3_user_settings.mako +++ b/addons/s3/templates/s3_user_settings.mako @@ -6,7 +6,7 @@ <%include file="s3_credentials_modal.mako"/>

- + ${addon_full_name} icon ${_("Connect or Reauthorize Account")} @@ -21,7 +21,7 @@ - + @@ -43,4 +43,4 @@ - + \ No newline at end of file diff --git a/addons/s3/tests/factories.py b/addons/s3/tests/factories.py index a8b49856d47..bafdbaf9c84 100644 --- a/addons/s3/tests/factories.py +++ b/addons/s3/tests/factories.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Factories for the S3 addon.""" import factory from factory.django import DjangoModelFactory @@ -11,9 +10,9 @@ class S3AccountFactory(ExternalAccountFactory): provider = 's3' - provider_id = factory.Sequence(lambda n: 'id-{0}'.format(n)) - oauth_key = factory.Sequence(lambda n: 'key-{0}'.format(n)) - oauth_secret = factory.Sequence(lambda n: 'secret-{0}'.format(n)) + provider_id = factory.Sequence(lambda n: f'id-{n}') + oauth_key = factory.Sequence(lambda n: f'key-{n}') + oauth_secret = factory.Sequence(lambda n: f'secret-{n}') display_name = 'S3 Fake User' diff --git a/addons/s3/tests/test_model.py b/addons/s3/tests/test_model.py index b616a66c652..8e549c5d1ae 100644 --- a/addons/s3/tests/test_model.py +++ b/addons/s3/tests/test_model.py @@ -1,7 +1,4 @@ -# from nose.tools import * # noqa -import mock -from nose.tools import (assert_false, assert_true, - assert_equal, assert_is_none) +from unittest import mock import pytest import unittest @@ -38,26 +35,33 @@ class TestNodeSettings(OAuthAddonNodeSettingsTestSuiteMixin, unittest.TestCase): NodeSettingsClass = NodeSettings UserSettingsFactory = S3UserSettingsFactory + def _node_settings_class_kwargs(self, node, user_settings): + return { + 'user_settings': self.user_settings, + 'folder_id': 'bucket_name:/path_goes_here/with_folder_id', + 'owner': self.node + } + def test_registration_settings(self): registration = ProjectFactory() clone, message = self.node_settings.after_register( self.node, registration, self.user, ) - assert_is_none(clone) + assert clone is None def test_before_register_no_settings(self): self.node_settings.user_settings = None message = self.node_settings.before_register(self.node, self.user) - assert_false(message) + assert not message def test_before_register_no_auth(self): self.node_settings.external_account = None message = self.node_settings.before_register(self.node, self.user) - assert_false(message) + assert not message def test_before_register_settings_and_auth(self): message = self.node_settings.before_register(self.node, self.user) - assert_true(message) + assert message @mock.patch('website.archiver.tasks.archive') def test_does_not_get_copied_to_registrations(self, mock_archive): @@ -66,7 +70,7 @@ def test_does_not_get_copied_to_registrations(self, mock_archive): auth=Auth(user=self.user), draft_registration=DraftRegistrationFactory(branched_from=self.node), ) - assert_false(registration.has_addon('s3')) + assert not registration.has_addon('s3') ## Overrides ## @@ -78,7 +82,7 @@ def test_serialize_credentials(self): expected = {'access_key': self.node_settings.external_account.oauth_key, 'secret_key': self.node_settings.external_account.oauth_secret} - assert_equal(credentials, expected) + assert credentials == expected @mock.patch('addons.s3.models.bucket_exists') @mock.patch('addons.s3.models.get_bucket_location_or_error') @@ -89,13 +93,50 @@ def test_set_folder(self, mock_location, mock_exists): self.node_settings.set_folder(folder_id, auth=Auth(self.user)) self.node_settings.save() # Bucket was set - assert_equal(self.node_settings.folder_id, folder_id) + assert self.node_settings.folder_id == folder_id # Log was saved last_log = self.node.logs.latest() - assert_equal(last_log.action, '{0}_bucket_linked'.format(self.short_name)) + assert last_log.action == f'{self.short_name}_bucket_linked' + + @mock.patch('addons.s3.models.bucket_exists') + def test_set_folder_nonexistent_bucket(self, mock_exists): + from addons.base import exceptions + mock_exists.return_value = False + with pytest.raises(exceptions.InvalidFolderError): + self.node_settings.set_folder('nonexistent-bucket:/', auth=Auth(self.user)) + + @mock.patch('addons.s3.models.bucket_exists') + @mock.patch('addons.s3.models.get_bucket_location_or_error') + def test_set_folder_invalid_auth(self, mock_location, mock_exists): + from addons.base.exceptions import InvalidAuthError + mock_exists.return_value = True + mock_location.side_effect = InvalidAuthError() + with pytest.raises(InvalidAuthError): + self.node_settings.set_folder('valid-bucket:/', auth=Auth(self.user)) def test_serialize_settings(self): settings = self.node_settings.serialize_waterbutler_settings() - expected = {'bucket': self.node_settings.folder_id, - 'encrypt_uploads': self.node_settings.encrypt_uploads} - assert_equal(settings, expected) + expected = { + 'bucket': 'bucket_name', + 'encrypt_uploads': self.node_settings.encrypt_uploads, + 'id': 'bucket_name:/path_goes_here/with_folder_id' + } + assert settings == expected + + def test_serialize_settings_supports_legacy_and_prefixed_folder_ids(self): + cases = [ + ('bucket', 'bucket'), + ('bucket:/', 'bucket'), + ('bucket:/prefix/', 'bucket'), + ] + + for folder_id, expected_bucket in cases: + self.node_settings.folder_id = folder_id + + settings = self.node_settings.serialize_waterbutler_settings() + + assert settings == { + 'bucket': expected_bucket, + 'encrypt_uploads': self.node_settings.encrypt_uploads, + 'id': folder_id, + } diff --git a/addons/s3/tests/test_serializer.py b/addons/s3/tests/test_serializer.py index a1c637020c3..f1fa025d49c 100644 --- a/addons/s3/tests/test_serializer.py +++ b/addons/s3/tests/test_serializer.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- """Serializer tests for the S3 addon.""" -import mock +from unittest import mock import pytest from addons.base.tests.serializers import StorageAddonSerializerTestSuiteMixin @@ -24,8 +23,8 @@ def setUp(self): self.mock_can_list = mock.patch('addons.s3.serializer.utils.can_list') self.mock_can_list.return_value = True self.mock_can_list.start() - super(TestS3Serializer, self).setUp() + super().setUp() def tearDown(self): self.mock_can_list.stop() - super(TestS3Serializer, self).tearDown() + super().tearDown() diff --git a/addons/s3/tests/test_view.py b/addons/s3/tests/test_view.py index 8db2b2a9091..c60d796b91d 100644 --- a/addons/s3/tests/test_view.py +++ b/addons/s3/tests/test_view.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- from rest_framework import status as http_status -from boto.exception import S3ResponseError -import mock -from nose.tools import (assert_equal, assert_equals, - assert_true, assert_in, assert_false) +from botocore.exceptions import NoCredentialsError +from unittest import mock import pytest from framework.auth import Auth @@ -18,6 +15,7 @@ from addons.s3.utils import validate_bucket_name, validate_bucket_location from website.util import api_url_for from admin.rdm_addons.utils import get_rdm_addon_option +from osf.models import ExternalAccount pytestmark = pytest.mark.django_db @@ -26,19 +24,19 @@ def setUp(self): self.mock_can_list = mock.patch('addons.s3.views.utils.can_list') self.mock_can_list.return_value = True self.mock_can_list.start() - self.mock_uid = mock.patch('addons.s3.views.utils.get_user_info') - self.mock_uid.return_value = {'id': '1234567890', 'display_name': 's3.user'} + self.mock_uid = mock.patch('addons.s3.views.utils.get_user_info', + return_value=mock.Mock(id='1234567890', display_name='s3.user')) self.mock_uid.start() self.mock_exists = mock.patch('addons.s3.views.utils.bucket_exists') self.mock_exists.return_value = True self.mock_exists.start() - super(TestS3Views, self).setUp() + super().setUp() def tearDown(self): self.mock_can_list.stop() self.mock_uid.stop() self.mock_exists.stop() - super(TestS3Views, self).tearDown() + super().tearDown() def test_s3_settings_input_empty_keys(self): url = self.project.api_url_for('s3_add_user_account') @@ -46,8 +44,8 @@ def test_s3_settings_input_empty_keys(self): 'access_key': '', 'secret_key': '' }, auth=self.user.auth, expect_errors=True) - assert_equals(rv.status_int, http_status.HTTP_400_BAD_REQUEST) - assert_in('All the fields above are required.', rv.body.decode()) + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST + assert 'All the fields above are required.' in rv.text def test_s3_settings_input_empty_access_key(self): url = self.project.api_url_for('s3_add_user_account') @@ -55,8 +53,8 @@ def test_s3_settings_input_empty_access_key(self): 'access_key': '', 'secret_key': 'Non-empty-secret-key' }, auth=self.user.auth, expect_errors=True) - assert_equals(rv.status_int, http_status.HTTP_400_BAD_REQUEST) - assert_in('All the fields above are required.', rv.body.decode()) + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST + assert 'All the fields above are required.' in rv.text def test_s3_settings_input_empty_secret_key(self): url = self.project.api_url_for('s3_add_user_account') @@ -64,8 +62,8 @@ def test_s3_settings_input_empty_secret_key(self): 'access_key': 'Non-empty-access-key', 'secret_key': '' }, auth=self.user.auth, expect_errors=True) - assert_equals(rv.status_int, http_status.HTTP_400_BAD_REQUEST) - assert_in('All the fields above are required.', rv.body.decode()) + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST + assert 'All the fields above are required.' in rv.text def test_s3_settings_rdm_addons_denied(self): institution = InstitutionFactory() @@ -75,12 +73,69 @@ def test_s3_settings_rdm_addons_denied(self): rdm_addon_option.is_allowed = False rdm_addon_option.save() url = self.project.api_url_for('s3_add_user_account') - rv = self.app.post_json(url,{ + rv = self.app.post_json(url, { 'access_key': 'aldkjf', 'secret_key': 'las' }, auth=self.user.auth, expect_errors=True) - assert_equal(rv.status_int, http_status.HTTP_403_FORBIDDEN) - assert_in(b'You are prohibited from using this add-on.', rv.body) + assert rv.status_code == http_status.HTTP_403_FORBIDDEN + assert 'You are prohibited from using this add-on.' in rv.text + + # G5: GRDM-53044 provider_id = "{aws_account_id}\t{access_key}" + # setUp already patches get_user_info/can_list as truthy MagicMocks. + # Note: oauth_key/oauth_secret are EncryptedTextField (JWE, non-deterministic), + # so we query by provider_id (plain CharField) and verify via attribute access. + def test_provider_id_format(self): + url = self.project.api_url_for('s3_add_user_account') + access_key = 'AKIAIOSFODNN7EXAMPLE' + rv = self.app.post_json(url, { + 'access_key': access_key, + 'secret_key': 'wJalrXUtnFEMI/K7MDENG', + }, auth=self.user.auth) + assert rv.status_code == http_status.HTTP_200_OK + # provider_id is not encrypted — safe to query + provider_id = f'1234567890\t{access_key}' + account = ExternalAccount.objects.get(provider='s3', provider_id=provider_id) + # provider_id must be "{aws_account_id}\t{access_key}" (GRDM-53044) + assert account.provider_id.endswith(f'\t{access_key}') + assert '\t' in account.provider_id + assert account.oauth_key == access_key + + def test_provider_id_dedup(self): + url = self.project.api_url_for('s3_add_user_account') + access_key = 'AKIAIOSFODNN7EXAMPLE' + provider_id = f'1234567890\t{access_key}' + rv1 = self.app.post_json(url, { + 'access_key': access_key, + 'secret_key': 'secret1', + }, auth=self.user.auth) + assert rv1.status_code == http_status.HTTP_200_OK + # Same AWS account (same provider_id) with new secret → update, no duplicate + rv2 = self.app.post_json(url, { + 'access_key': access_key, + 'secret_key': 'secret2', + }, auth=self.user.auth) + assert rv2.status_code == http_status.HTTP_200_OK + # Query by provider_id (plain CharField) — oauth_key is encrypted (JWE) + accounts = ExternalAccount.objects.filter(provider='s3', provider_id=provider_id) + assert accounts.count() == 1 + assert accounts.first().oauth_secret == 'secret2' + + # G1: missing key field → KeyError → 400 + def test_s3_settings_missing_key_field(self): + url = self.project.api_url_for('s3_add_user_account') + rv = self.app.post_json(url, {}, auth=self.user.auth, expect_errors=True) + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST + + # G2: get_user_info returns None → 400 + "Unable to access account." + @mock.patch('addons.s3.views.utils.get_user_info', return_value=None) + def test_s3_settings_invalid_account(self, mock_get_user_info): + url = self.project.api_url_for('s3_add_user_account') + rv = self.app.post_json(url, { + 'access_key': 'aldkjf', + 'secret_key': 'las', + }, auth=self.user.auth, expect_errors=True) + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST + assert 'Unable to access account.' in rv.text def test_s3_set_bucket_no_settings(self): user = AuthUserFactory() @@ -88,9 +143,9 @@ def test_s3_set_bucket_no_settings(self): url = self.project.api_url_for('s3_set_config') res = self.app.put_json( url, {'s3_bucket': 'hammertofall'}, auth=user.auth, - expect_errors=True + expect_errors=True, ) - assert_equal(res.status_code, http_status.HTTP_400_BAD_REQUEST) + assert res.status_code == http_status.HTTP_400_BAD_REQUEST def test_s3_set_bucket_no_auth(self): @@ -100,9 +155,9 @@ def test_s3_set_bucket_no_auth(self): url = self.project.api_url_for('s3_set_config') res = self.app.put_json( url, {'s3_bucket': 'hammertofall'}, auth=user.auth, - expect_errors=True + expect_errors=True, ) - assert_equal(res.status_code, http_status.HTTP_403_FORBIDDEN) + assert res.status_code == http_status.HTTP_403_FORBIDDEN def test_s3_set_bucket_registered(self): registration = self.project.register_node( @@ -115,7 +170,7 @@ def test_s3_set_bucket_registered(self): expect_errors=True, ) - assert_equal(res.status_code, http_status.HTTP_400_BAD_REQUEST) + assert res.status_code == http_status.HTTP_400_BAD_REQUEST @mock.patch('addons.s3.views.utils.can_list', return_value=False) def test_user_settings_cant_list(self, mock_can_list): @@ -125,20 +180,20 @@ def test_user_settings_cant_list(self, mock_can_list): 'secret_key': 'las' }, auth=self.user.auth, expect_errors=True) - assert_in('Unable to list buckets.', rv.body.decode()) - assert_equals(rv.status_int, http_status.HTTP_400_BAD_REQUEST) + assert 'Unable to list buckets.' in rv.text + assert rv.status_code == http_status.HTTP_400_BAD_REQUEST def test_s3_remove_node_settings_owner(self): url = self.node_settings.owner.api_url_for('s3_deauthorize_node') self.app.delete(url, auth=self.user.auth) result = self.Serializer().serialize_settings(node_settings=self.node_settings, current_user=self.user) - assert_equal(result['nodeHasAuth'], False) + assert result['nodeHasAuth'] == False def test_s3_remove_node_settings_unauthorized(self): url = self.node_settings.owner.api_url_for('s3_deauthorize_node') ret = self.app.delete(url, auth=None, expect_errors=True) - assert_equal(ret.status_code, 401) + assert ret.status_code == 401 def test_s3_get_node_settings_owner(self): self.node_settings.set_auth(self.external_account, self.user) @@ -148,23 +203,23 @@ def test_s3_get_node_settings_owner(self): res = self.app.get(url, auth=self.user.auth) result = res.json['result'] - assert_equal(result['nodeHasAuth'], True) - assert_equal(result['userIsOwner'], True) - assert_equal(result['folder']['path'], self.node_settings.folder_id) + assert result['nodeHasAuth'] == True + assert result['userIsOwner'] == True + assert result['folder']['path'] == self.node_settings.folder_id def test_s3_get_node_settings_unauthorized(self): url = self.node_settings.owner.api_url_for('s3_get_config') unauthorized = AuthUserFactory() ret = self.app.get(url, auth=unauthorized.auth, expect_errors=True) - assert_equal(ret.status_code, 403) + assert ret.status_code == 403 ## Overrides ## @mock.patch('addons.s3.models.get_bucket_names') def test_folder_list(self, mock_names): mock_names.return_value = ['bucket1', 'bucket2'] - super(TestS3Views, self).test_folder_list() + super().test_folder_list() @mock.patch('addons.s3.models.bucket_exists') @mock.patch('addons.s3.models.get_bucket_location_or_error') @@ -172,25 +227,22 @@ def test_set_config(self, mock_location, mock_exists): mock_exists.return_value = True mock_location.return_value = '' self.node_settings.set_auth(self.external_account, self.user) - url = self.project.api_url_for('{0}_set_config'.format(self.ADDON_SHORT_NAME)) + url = self.project.api_url_for(f'{self.ADDON_SHORT_NAME}_set_config') res = self.app.put_json(url, { 'selected': self.folder }, auth=self.user.auth) - assert_equal(res.status_code, http_status.HTTP_200_OK) + assert res.status_code == http_status.HTTP_200_OK self.project.reload() self.node_settings.reload() - assert_equal( - self.project.logs.latest().action, - '{0}_bucket_linked'.format(self.ADDON_SHORT_NAME) - ) - assert_equal(res.json['result']['folder']['name'], self.node_settings.folder_name) + assert self.project.logs.latest().action == f'{self.ADDON_SHORT_NAME}_bucket_linked' + assert res.json['result']['folder']['name'] == self.node_settings.folder_name class TestCreateBucket(S3AddonTestCase, OsfTestCase): def setUp(self): - super(TestCreateBucket, self).setUp() + super().setUp() self.user = AuthUserFactory() self.consolidated_auth = Auth(user=self.user) @@ -212,57 +264,57 @@ def setUp(self): self.node_settings.save() def test_bad_names(self): - assert_false(validate_bucket_name('')) - assert_false(validate_bucket_name('no')) - assert_false(validate_bucket_name('a' * 64)) - assert_false(validate_bucket_name(' leadingspace')) - assert_false(validate_bucket_name('trailingspace ')) - assert_false(validate_bucket_name('bogus naMe')) - assert_false(validate_bucket_name('.cantstartwithp')) - assert_false(validate_bucket_name('or.endwith.')) - assert_false(validate_bucket_name('..nodoubles')) - assert_false(validate_bucket_name('no_unders_in')) - assert_false(validate_bucket_name('-leadinghyphen')) - assert_false(validate_bucket_name('trailinghyphen-')) - assert_false(validate_bucket_name('Mixedcase')) - assert_false(validate_bucket_name('empty..label')) - assert_false(validate_bucket_name('label-.trailinghyphen')) - assert_false(validate_bucket_name('label.-leadinghyphen')) - assert_false(validate_bucket_name('8.8.8.8')) - assert_false(validate_bucket_name('600.9000.0.28')) - assert_false(validate_bucket_name('no_underscore')) - assert_false(validate_bucket_name('_nounderscoreinfront')) - assert_false(validate_bucket_name('no-underscore-in-back_')) - assert_false(validate_bucket_name('no-underscore-in_the_middle_either')) + assert not validate_bucket_name('') + assert not validate_bucket_name('no') + assert not validate_bucket_name('a' * 64) + assert not validate_bucket_name(' leadingspace') + assert not validate_bucket_name('trailingspace ') + assert not validate_bucket_name('bogus naMe') + assert not validate_bucket_name('.cantstartwithp') + assert not validate_bucket_name('or.endwith.') + assert not validate_bucket_name('..nodoubles') + assert not validate_bucket_name('no_unders_in') + assert not validate_bucket_name('-leadinghyphen') + assert not validate_bucket_name('trailinghyphen-') + assert not validate_bucket_name('Mixedcase') + assert not validate_bucket_name('empty..label') + assert not validate_bucket_name('label-.trailinghyphen') + assert not validate_bucket_name('label.-leadinghyphen') + assert not validate_bucket_name('8.8.8.8') + assert not validate_bucket_name('600.9000.0.28') + assert not validate_bucket_name('no_underscore') + assert not validate_bucket_name('_nounderscoreinfront') + assert not validate_bucket_name('no-underscore-in-back_') + assert not validate_bucket_name('no-underscore-in_the_middle_either') def test_names(self): - assert_true(validate_bucket_name('imagoodname')) - assert_true(validate_bucket_name('still.passing')) - assert_true(validate_bucket_name('can-have-dashes')) - assert_true(validate_bucket_name('kinda.name.spaced')) - assert_true(validate_bucket_name('a-o.valid')) - assert_true(validate_bucket_name('11.12.m')) - assert_true(validate_bucket_name('a--------a')) - assert_true(validate_bucket_name('a' * 63)) + assert validate_bucket_name('imagoodname') + assert validate_bucket_name('still.passing') + assert validate_bucket_name('can-have-dashes') + assert validate_bucket_name('kinda.name.spaced') + assert validate_bucket_name('a-o.valid') + assert validate_bucket_name('11.12.m') + assert validate_bucket_name('a--------a') + assert validate_bucket_name('a' * 63) def test_bad_locations(self): - assert_false(validate_bucket_location('Venus')) - assert_false(validate_bucket_location('AlphaCentari')) - assert_false(validate_bucket_location('CostaRica')) + assert not validate_bucket_location('Venus') + assert not validate_bucket_location('AlphaCentauri') + assert not validate_bucket_location('CostaRica') def test_locations(self): - assert_true(validate_bucket_location('')) - assert_true(validate_bucket_location('eu-central-1')) - assert_true(validate_bucket_location('ca-central-1')) - assert_true(validate_bucket_location('us-west-1')) - assert_true(validate_bucket_location('us-west-2')) - assert_true(validate_bucket_location('ap-northeast-1')) - assert_true(validate_bucket_location('ap-northeast-2')) - assert_true(validate_bucket_location('ap-southeast-1')) - assert_true(validate_bucket_location('ap-southeast-2')) - assert_true(validate_bucket_location('sa-east-1')) - assert_true(validate_bucket_location('eu-west-1')) - assert_true(validate_bucket_location('eu-west-2')) + assert validate_bucket_location('') + assert validate_bucket_location('eu-central-1') + assert validate_bucket_location('ca-central-1') + assert validate_bucket_location('us-west-1') + assert validate_bucket_location('us-west-2') + assert validate_bucket_location('ap-northeast-1') + assert validate_bucket_location('ap-northeast-2') + assert validate_bucket_location('ap-southeast-1') + assert validate_bucket_location('ap-southeast-2') + assert validate_bucket_location('sa-east-1') + assert validate_bucket_location('eu-west-1') + assert validate_bucket_location('eu-west-2') @mock.patch('addons.s3.views.utils.create_bucket') @mock.patch('addons.s3.views.utils.get_bucket_names') @@ -283,23 +335,23 @@ def test_create_bucket_pass(self, mock_names, mock_make): auth=self.user.auth ) - assert_equal(ret.status_int, http_status.HTTP_200_OK) - assert_equal(ret.json, {}) + assert ret.status_code == http_status.HTTP_200_OK + assert ret.json == {} @mock.patch('addons.s3.views.utils.create_bucket') def test_create_bucket_fail(self, mock_make): - error = S3ResponseError(418, 'because Im a test') - error.message = 'This should work' + error = NoCredentialsError(operation_name='create_bucket') mock_make.side_effect = error - url = '/api/v1/project/{0}/s3/newbucket/'.format(self.project._id) - ret = self.app.post_json(url, {'bucket_name': 'doesntevenmatter'}, auth=self.user.auth, expect_errors=True) + url = f'/api/v1/project/{self.project._id}/s3/newbucket/' + ret = self.app.post_json(url, {'bucket_name': 'doesntevenmatter'}, auth=self.user.auth, + expect_errors=True) - assert_equals(ret.body.decode(), '{"message": "This should work", "title": "Problem connecting to S3"}') + assert ret.text == '{"message": "Unable to locate credentials", "title": "Problem connecting to S3"}' @mock.patch('addons.s3.views.utils.create_bucket') def test_bad_location_fails(self, mock_make): - url = '/api/v1/project/{0}/s3/newbucket/'.format(self.project._id) + url = f'/api/v1/project/{self.project._id}/s3/newbucket/' ret = self.app.post_json( url, { @@ -307,6 +359,151 @@ def test_bad_location_fails(self, mock_make): 'bucket_location': 'not a real bucket location', }, auth=self.user.auth, - expect_errors=True) + expect_errors=True, + ) + assert ret.text == '{"message": "That bucket location is not valid.", "title": "Invalid bucket location"}' + + # G4: invalid bucket name → 400 (name check fires before any S3 call) + def test_create_bucket_bad_name(self): + url = f'/api/v1/project/{self.project._id}/s3/newbucket/' + ret = self.app.post_json( + url, + { + 'bucket_name': 'no', + 'bucket_location': '', + }, + auth=self.user.auth, + expect_errors=True, + ) + assert ret.status_code == http_status.HTTP_400_BAD_REQUEST + assert 'That bucket name is not valid.' in ret.text + + @mock.patch('addons.s3.views.utils.create_bucket') + def test_create_bucket_client_error(self, mock_make): + from botocore.exceptions import ClientError + error = ClientError( + {'Error': {'Code': 'BucketAlreadyExists', 'Message': 'The requested bucket name is not available.'}}, + 'CreateBucket' + ) + mock_make.side_effect = error + url = f'/api/v1/project/{self.project._id}/s3/newbucket/' + ret = self.app.post_json( + url, + {'bucket_name': 'valid-bucket-name', 'bucket_location': ''}, + auth=self.user.auth, + expect_errors=True, + ) + assert ret.status_code == http_status.HTTP_400_BAD_REQUEST + assert ret.json['title'] == "Problem creating bucket 'valid-bucket-name'" + assert 'BucketAlreadyExists' in ret.json['message'] + + @mock.patch('addons.s3.views.utils.create_bucket') + def test_create_bucket_boto_core_error(self, mock_make): + # BotoCoreError covers the non-response failures (bad parameters, endpoint + # unreachable, ...). Without the catchall these surface as a 500. + from botocore.exceptions import ParamValidationError + mock_make.side_effect = ParamValidationError(report='Invalid bucket configuration') + url = f'/api/v1/project/{self.project._id}/s3/newbucket/' + ret = self.app.post_json( + url, + {'bucket_name': 'valid-bucket-name', 'bucket_location': ''}, + auth=self.user.auth, + expect_errors=True, + ) + assert ret.status_code == http_status.HTTP_400_BAD_REQUEST + assert ret.json['title'] == 'Error connecting to S3' + assert 'Invalid bucket configuration' in ret.json['message'] + + +class TestS3Utils: + """Pure unit tests for utility functions — no DB required.""" + + def test_owner_from_dict_without_display_name(self): + from addons.s3.utils import Owner + owner = Owner.from_dict({'ID': 'owner-id'}) + assert owner.id == 'owner-id' + assert owner.display_name == 'owner-id' + + # G7: bucket_exists with empty name → False (early exit, no S3 call) + def test_bucket_exists_empty_name(self): + from addons.s3.utils import bucket_exists + assert bucket_exists('key', 'secret', '') is False + + # G8: can_list(None, None) → False (early exit, no S3 call) + def test_can_list_no_keys(self): + from addons.s3.utils import can_list + assert can_list(None, None) is False + + # G9: us-east-1 is reported as a null LocationConstraint, but BUCKET_LOCATIONS + # keys that region as ''. Passing the null through labels the bucket '(None)'. + @mock.patch('addons.s3.utils.connect_s3') + def test_null_location_constraint_becomes_empty_string(self, mock_connect): + from addons.s3.utils import get_bucket_location_or_error + mock_connect.return_value.get_bucket_location.return_value = {'LocationConstraint': None} + assert get_bucket_location_or_error('key', 'secret', 'bucket') == '' + + # G10: every other region reports its own name and must survive untouched + @mock.patch('addons.s3.utils.connect_s3') + def test_named_region_is_passed_through(self, mock_connect): + from addons.s3.utils import get_bucket_location_or_error + mock_connect.return_value.get_bucket_location.return_value = { + 'LocationConstraint': 'ap-northeast-1' + } + assert get_bucket_location_or_error('key', 'secret', 'bucket') == 'ap-northeast-1' + + # G11: the queried prefix comes back in CommonPrefixes and must not list itself + @mock.patch('addons.s3.utils.boto3.client') + def test_get_bucket_prefixes_skips_the_queried_prefix(self, mock_client): + from addons.s3.utils import get_bucket_prefixes + mock_client.return_value.list_objects.return_value = { + 'CommonPrefixes': [ + {'Prefix': 'parent/'}, + {'Prefix': 'parent/child/'}, + ] + } + + folders = get_bucket_prefixes('key', 'secret', 'parent/', 'bucket') + + assert len(folders) == 1 + assert folders[0]['name'] == 'child' + assert folders[0]['id'] == 'bucket:/parent/child/' + assert folders[0]['path'] == 'parent/child/' + assert folders[0]['kind'] == 'folder' + + # G12: a bucket with no sub-prefixes omits CommonPrefixes entirely + @mock.patch('addons.s3.utils.boto3.client') + def test_get_bucket_prefixes_no_common_prefixes(self, mock_client): + from addons.s3.utils import get_bucket_prefixes + mock_client.return_value.list_objects.return_value = {} + assert get_bucket_prefixes('key', 'secret', '', 'bucket') == [] + + # G13: boto failures must become HTTPError, not an uncaught 500 + @mock.patch('addons.s3.utils.boto3.client') + def test_get_bucket_prefixes_missing_credentials_raises_403(self, mock_client): + from framework.exceptions import HTTPError + from addons.s3.utils import get_bucket_prefixes + mock_client.return_value.list_objects.side_effect = NoCredentialsError() + + with pytest.raises(HTTPError) as exc_info: + get_bucket_prefixes('key', 'secret', '', 'bucket') + + assert exc_info.value.code == http_status.HTTP_403_FORBIDDEN + + # G14: a ClientError keeps whatever status S3 itself returned + @mock.patch('addons.s3.utils.boto3.client') + def test_get_bucket_prefixes_client_error_keeps_status(self, mock_client): + from botocore.exceptions import ClientError + from framework.exceptions import HTTPError + from addons.s3.utils import get_bucket_prefixes + mock_client.return_value.list_objects.side_effect = ClientError( + { + 'Error': {'Code': 'NoSuchBucket', 'Message': 'denied'}, + 'ResponseMetadata': {'HTTPStatusCode': http_status.HTTP_404_NOT_FOUND}, + }, + 'ListObjects', + ) + + with pytest.raises(HTTPError) as exc_info: + get_bucket_prefixes('key', 'secret', '', 'bucket') - assert_equals(ret.body.decode(), '{"message": "That bucket location is not valid.", "title": "Invalid bucket location"}') + assert exc_info.value.code == http_status.HTTP_404_NOT_FOUND diff --git a/addons/s3/tests/utils.py b/addons/s3/tests/utils.py index 15b0153f23b..e83f4ebf489 100644 --- a/addons/s3/tests/utils.py +++ b/addons/s3/tests/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from addons.base.tests.base import OAuthAddonTestCaseMixin, AddonTestCase from addons.s3.tests.factories import S3AccountFactory from addons.s3.provider import S3Provider diff --git a/addons/s3/utils.py b/addons/s3/utils.py index 0738b5bf397..f0cb70da46c 100644 --- a/addons/s3/utils.py +++ b/addons/s3/utils.py @@ -1,14 +1,19 @@ import re +import logging +from dataclasses import dataclass +from typing import Optional # Python 3.6 does not support | operator for type hints + from rest_framework import status as http_status -from boto import exception -from boto.s3.connection import S3Connection -from boto.s3.connection import OrdinaryCallingFormat +import boto3 +from botocore import exceptions from framework.exceptions import HTTPError from addons.base.exceptions import InvalidAuthError, InvalidFolderError from addons.s3.settings import BUCKET_LOCATIONS +logger = logging.getLogger(__name__) + def connect_s3(access_key=None, secret_key=None, node_settings=None): """Helper to build an S3Connection object @@ -18,19 +23,27 @@ def connect_s3(access_key=None, secret_key=None, node_settings=None): if node_settings is not None: if node_settings.external_account is not None: access_key, secret_key = node_settings.external_account.oauth_key, node_settings.external_account.oauth_secret - connection = S3Connection(access_key, secret_key, calling_format=OrdinaryCallingFormat()) + connection = boto3.client( + 's3', + aws_access_key_id=access_key, + aws_secret_access_key=secret_key + ) return connection +def get_status_for_error(e: exceptions.ClientError) -> int: + return e.response['ResponseMetadata']['HTTPStatusCode'] + + def get_bucket_names(node_settings): try: - buckets = connect_s3(node_settings=node_settings).get_all_buckets() - except exception.NoAuthHandlerFound: + response = connect_s3(node_settings=node_settings).list_buckets() + except exceptions.NoCredentialsError: raise HTTPError(http_status.HTTP_403_FORBIDDEN) - except exception.BotoServerError as e: - raise HTTPError(e.status) + except exceptions.ClientError as e: + raise HTTPError(get_status_for_error(e)) - return [bucket.name for bucket in buckets] + return [bucket['Name'] for bucket in response['Buckets']] def validate_bucket_location(location): @@ -51,28 +64,32 @@ def validate_bucket_name(name): def create_bucket(node_settings, bucket_name, location=''): - return connect_s3(node_settings=node_settings).create_bucket(bucket_name, location=location) + client = connect_s3(node_settings=node_settings) + + # default bucket location won't work with location constraint + if not location or location == 'us-east-1': + return client.create_bucket(Bucket=bucket_name) + else: + return client.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration={ + 'LocationConstraint': location, + } + ) def bucket_exists(access_key, secret_key, bucket_name): - """Tests for the existance of a bucket and if the user + """Tests for the existence of a bucket and if the user can access it with the given keys """ if not bucket_name: return False - connection = connect_s3(access_key, secret_key) - - if bucket_name != bucket_name.lower(): - # Must use ordinary calling format for mIxEdCaSe bucket names - # otherwise use the default as it handles bucket outside of the US - connection.calling_format = OrdinaryCallingFormat() - try: # Will raise an exception if bucket_name doesn't exist - connect_s3(access_key, secret_key).head_bucket(bucket_name) - except exception.S3ResponseError as e: - if e.status not in (301, 302): + connect_s3(access_key, secret_key).head_bucket(Bucket=bucket_name) + except exceptions.ClientError as e: + if get_status_for_error(e) not in (301, 302): return False return True @@ -87,38 +104,78 @@ def can_list(access_key, secret_key): return False try: - connect_s3(access_key, secret_key).get_all_buckets() - except exception.S3ResponseError: + connect_s3(access_key, secret_key).list_buckets() + except exceptions.ClientError: return False return True -def get_user_info(access_key, secret_key): + +@dataclass(frozen=True) # slots is not supported in python 3.6 +class Owner: + __slots__ = ['display_name', 'id'] + + display_name: str + id: str + + @classmethod + def from_dict(cls, data): + owner_id = data['ID'] + return cls(data.get('DisplayName') or owner_id, owner_id) + + +def get_user_info(access_key: str, secret_key: str) -> Optional[Owner]: """Returns an S3 User with .display_name and .id, or None """ if not (access_key and secret_key): return None try: - return connect_s3(access_key, secret_key).get_all_buckets().owner - except exception.S3ResponseError: + return Owner.from_dict(connect_s3(access_key, secret_key).list_buckets()['Owner']) + except exceptions.ClientError: return None - return None + def get_bucket_location_or_error(access_key, secret_key, bucket_name): """Returns the location of a bucket or raises AddonError """ try: - connection = connect_s3(access_key, secret_key) - except Exception: + # Will raise an exception if bucket_name doesn't exist. + # us-east-1 is reported as a null LocationConstraint; BUCKET_LOCATIONS keys it as ''. + return connect_s3(access_key, secret_key).get_bucket_location(Bucket=bucket_name)['LocationConstraint'] or '' + except exceptions.NoCredentialsError: raise InvalidAuthError() + except exceptions.ClientError: + raise InvalidFolderError() - if bucket_name != bucket_name.lower() or '.' in bucket_name: - # Must use ordinary calling format for mIxEdCaSe bucket names - # otherwise use the default as it handles bucket outside of the US - connection.calling_format = OrdinaryCallingFormat() + +def get_bucket_prefixes(access_key, secret_key, prefix, bucket_name): + s3 = boto3.client( + 's3', + aws_access_key_id=access_key, + aws_secret_access_key=secret_key + ) try: - # Will raise an exception if bucket_name doesn't exist - return connect_s3(access_key, secret_key).get_bucket(bucket_name, validate=False).get_location() - except exception.S3ResponseError: - raise InvalidFolderError() + result = s3.list_objects(Bucket=bucket_name, Prefix=prefix, Delimiter='/') + except exceptions.NoCredentialsError: + raise HTTPError(http_status.HTTP_403_FORBIDDEN) + except exceptions.ClientError as e: + raise HTTPError(get_status_for_error(e)) + + folders = [] + for common_prefixes in result.get('CommonPrefixes', []): + key_name = common_prefixes.get('Prefix') + if key_name != prefix: + folders.append( + { + 'path': key_name, + 'id': f'{bucket_name}:/{key_name}', + 'folder_id': key_name, + 'kind': 'folder', + 'bucket_name': bucket_name, + 'name': key_name.split('/')[-2], + 'addon': 's3', + } + ) + + return folders diff --git a/addons/s3/views.py b/addons/s3/views.py index 672dd335305..5bccd9b3d3a 100644 --- a/addons/s3/views.py +++ b/addons/s3/views.py @@ -1,7 +1,8 @@ from rest_framework import status as http_status -from boto import exception +from botocore import exceptions from django.core.exceptions import ValidationError +from django.db import IntegrityError, transaction from flask import request from framework.exceptions import HTTPError @@ -58,7 +59,9 @@ def _set_folder(node_addon, folder, auth): def s3_folder_list(node_addon, **kwargs): """ Returns all the subsequent folders under the folder id passed. """ - return node_addon.get_folders() + path = request.args.get('path', '') + folder_id = request.args.get('id', '') + return node_addon.get_folders(path=path, folder_id=folder_id) @must_be_logged_in @must_be_rdm_addons_allowed(SHORT_NAME) @@ -95,16 +98,17 @@ def s3_add_user_account(auth, **kwargs): masked_access_key = f'****{access_key[-4:]}' if len(access_key) > 4 else '****' display_name = f'{user_info.display_name} ({masked_access_key})' try: - account = ExternalAccount( - provider=SHORT_NAME, - provider_name=FULL_NAME, - oauth_key=access_key, - oauth_secret=secret_key, - provider_id=provider_id, - display_name=display_name, - ) - account.save() - except ValidationError: + with transaction.atomic(): + account = ExternalAccount( + provider=SHORT_NAME, + provider_name=FULL_NAME, + oauth_key=access_key, + oauth_secret=secret_key, + provider_id=provider_id, + display_name=display_name, + ) + account.save() + except (ValidationError, IntegrityError): # ... or get the old one account = ExternalAccount.objects.get( provider=SHORT_NAME, @@ -148,19 +152,21 @@ def create_bucket(auth, node_addon, **kwargs): try: utils.create_bucket(node_addon, bucket_name, bucket_location) - except exception.S3ResponseError as e: + except exceptions.NoCredentialsError as e: return { - 'message': e.message, + 'message': str(e), 'title': 'Problem connecting to S3', }, http_status.HTTP_400_BAD_REQUEST - except exception.S3CreateError as e: + except exceptions.ClientError as e: return { - 'message': e.message, - 'title': "Problem creating bucket '{0}'".format(bucket_name), + 'message': str(e), + 'title': f"Problem creating bucket '{bucket_name}'", }, http_status.HTTP_400_BAD_REQUEST - except exception.BotoClientError as e: # Base class catchall + except exceptions.BotoCoreError as e: + # Base class catchall. ClientError is not a BotoCoreError, and NoCredentialsError + # is handled above, so neither of the clauses above is shadowed by this one. return { - 'message': e.message, + 'message': str(e), 'title': 'Error connecting to S3', }, http_status.HTTP_400_BAD_REQUEST diff --git a/addons/weko/client.py b/addons/weko/client.py index f7b00fcf459..634f62f9cda 100644 --- a/addons/weko/client.py +++ b/addons/weko/client.py @@ -15,6 +15,19 @@ def _flatten_indices(indices): return r +def _get_truncated_index_ids(descs): + # WEKO replaces the children beyond an index's display number with a "more" entry + # unless the index is listed in more_ids + r = [] + for desc in descs: + if 'children' not in desc: + continue + if any(child['id'] == 'more' for child in desc['children']): + r.append(str(desc['id'])) + r += _get_truncated_index_ids(desc['children']) + return r + + def _is_valid_index(desc): if 'name' in desc and 'id' in desc: return True @@ -58,11 +71,24 @@ def get_login_user(self, default_user=None): results = resp.json().get('results', {}) return results.get('subitem_mail_address', default_user) - def get_indices(self): + def get_indices(self, accept_language=None): """ Get all indices from the WEKO. """ - root = self._get('api/tree?action=browsing') + headers = None if accept_language is None else {'Accept-Language': accept_language} + more_ids = [] + while True: + path = 'api/tree?action=browsing' + if more_ids: + path += '&more_ids=' + '/'.join(more_ids) + root = self._get(path, headers=headers) + truncated_ids = _get_truncated_index_ids(root) + if not truncated_ids: + break + unexpanded_ids = set(truncated_ids) & set(more_ids) + if unexpanded_ids: + raise ValueError(f'WEKO did not expand indices: {unexpanded_ids}') + more_ids += truncated_ids indices = [] for desc in root: if not _is_valid_index(desc): @@ -97,8 +123,11 @@ def _base_host(self): return self.host return self.host[:-6] - def _get(self, path): - resp = requests.get(self._base_host + path, **self._requests_args()) + def _get(self, path, headers=None): + resp = requests.get( + self._base_host + path, + **self._requests_args(headers=headers) + ) resp.raise_for_status() return resp.json() diff --git a/addons/weko/serializer.py b/addons/weko/serializer.py index a191f3f94b1..add5743edb4 100644 --- a/addons/weko/serializer.py +++ b/addons/weko/serializer.py @@ -1,6 +1,11 @@ +import logging + +from flask_babel import get_locale + from addons.base.serializer import OAuthAddonSerializer from . import settings as weko_settings from .apps import SHORT_NAME +from .client import _flatten_indices from website.util import api_url_for, web_url_for from admin.rdm_addons.utils import get_rdm_addon_option @@ -8,6 +13,9 @@ from requests import exceptions as requests_exceptions +logger = logging.getLogger(__name__) + + def get_repository_options(user): repos = list(weko_settings.REPOSITORY_IDS) for institution_id in user.affiliated_institutions.all(): @@ -90,10 +98,14 @@ def serialized_node_settings(self): if self.node_settings.has_auth: c = self.node_settings.create_client() indices = c.get_indices() + # WEKO names indices per language; `title` stays its default name because that is what gets saved. + # The browser's Accept-Language is not forwarded: WEKO matches tags exactly and answers "ja-JP" in English + localized_indices = c.get_indices(accept_language=get_locale().language) + localized_titles = {index.identifier: index.title for index in _flatten_indices(localized_indices)} result.update({ 'validCredentials': True, - 'indices': [self._serialize_index(index) for index in indices], + 'indices': [self._serialize_index(index, localized_titles) for index in indices], 'savedIndex': { 'title': self.node_settings.index_title, 'id': self.node_settings.index_id, @@ -136,9 +148,17 @@ def serialize_settings(self, node_settings, current_user, client=None): result.update(self.serialized_node_settings) return result - def _serialize_index(self, index): + def _serialize_index(self, index, localized_titles): + if index.identifier in localized_titles: + localized_title = localized_titles[index.identifier] + else: + # WEKO caches the tree per language and refreshes only the languages registered in its + # admin settings, so the two trees can disagree; the name is the only thing at stake + logger.warning('WEKO returned no localized name for index %s; showing its default name', index.identifier) + localized_title = index.title return { 'title': index.title, + 'localizedTitle': localized_title, 'id': index.identifier, - 'children': [self._serialize_index(i) for i in index.children], + 'children': [self._serialize_index(i, localized_titles) for i in index.children], } diff --git a/addons/weko/static/wekoNodeConfig.js b/addons/weko/static/wekoNodeConfig.js index 7b1fcd0a430..03b4b096c81 100644 --- a/addons/weko/static/wekoNodeConfig.js +++ b/addons/weko/static/wekoNodeConfig.js @@ -8,6 +8,7 @@ var bootbox = require('bootbox'); var Raven = require('raven-js'); var $osf = require('js/osfHelpers'); +require('js/rdmSelect2'); var $modal = $('#wekoInputCredentials'); @@ -29,31 +30,53 @@ function _getIndexById(indices, id) { return null; } -function _getIndexDisplayTitle(index, level) { - if (level === 0) { - return index.title; - } - var prefix = ''; - for (var i = 0; i < level; i ++) { - prefix += ' '; - } - return prefix + '- ' + index.title; -} - -function _flattenIndices(indices, level) { +function _flattenIndices(indices, ancestorTitles, ancestorLocalizedTitles) { const r = []; indices.forEach(function(data) { - const displayTitle = _getIndexDisplayTitle(data, level); + const titles = ancestorTitles.concat([data.title]); + const localizedTitles = ancestorLocalizedTitles.concat([data.localizedTitle]); r.push(Object.assign({ - displayTitle: displayTitle, + displayTitle: titles.join(' / '), + label: localizedTitles.join(' / ') + ' (ID:' + data.id + ')', }, data)); - _flattenIndices(data.children, level + 1).forEach(function(child) { + _flattenIndices(data.children, titles, localizedTitles).forEach(function(child) { r.push(child); }); }); return r; } +ko.bindingHandlers.wekoIndexSelect2 = { + // select2 mirrors the relies on it + $(element).select2({ + width: '100%', + // The + value: selectedIndexId, + wekoIndexSelect2: {value: selectedIndexId, labels: indexLabels}"> diff --git a/addons/weko/tests/test_client.py b/addons/weko/tests/test_client.py index d616196ec50..a624e293adc 100644 --- a/addons/weko/tests/test_client.py +++ b/addons/weko/tests/test_client.py @@ -19,6 +19,36 @@ def mock_requests_get(url, **kwargs): return utils.mock_response_404 +def _fake_truncated_indices(expanded_ids): + grandchildren = [{'id': '300', 'name': 'Grandchild 1', 'children': []}] + if '201' in expanded_ids: + grandchildren.append({'id': '301', 'name': 'Grandchild 2', 'children': []}) + else: + grandchildren.append({'id': 'more'}) + children = [{'id': '200', 'name': 'Child 1', 'children': []}] + if '100' in expanded_ids: + children.append({'id': '201', 'name': 'Child 2', 'children': grandchildren}) + else: + children.append({'id': 'more'}) + return [{'id': '100', 'name': 'Sample Index', 'children': children}] + + +def mock_requests_get_truncated(url, **kwargs): + prefix = 'https://test.sample.nii.ac.jp/api/tree?action=browsing' + if url == prefix: + return utils.MockResponse(_fake_truncated_indices([]), 200) + if url.startswith(prefix + '&more_ids='): + expanded_ids = url[len(prefix + '&more_ids='):].split('/') + return utils.MockResponse(_fake_truncated_indices(expanded_ids), 200) + return utils.mock_response_404 + + +def mock_requests_get_never_expanded(url, **kwargs): + if url.startswith('https://test.sample.nii.ac.jp/api/tree?action=browsing'): + return utils.MockResponse(_fake_truncated_indices([]), 200) + return utils.mock_response_404 + + class TestWEKOClient(OsfTestCase): def setUp(self): self.host = utils.fake_weko_host @@ -35,6 +65,37 @@ def test_weko_get_indices(self, get_req_mock): assert_equal(indices[0].title, 'Sample Index') assert_equal(indices[0].identifier, 100) + @mock.patch('requests.get', side_effect=mock_requests_get_truncated) + def test_weko_get_indices_expands_truncated_children(self, get_req_mock): + indices = self.conn.get_indices() + + titles = [i.title for i in client._flatten_indices(indices)] + assert_equal(titles, ['Sample Index', 'Child 1', 'Child 2', 'Grandchild 1', 'Grandchild 2']) + assert_equal( + [c[0][0] for c in get_req_mock.call_args_list], + [ + 'https://test.sample.nii.ac.jp/api/tree?action=browsing', + 'https://test.sample.nii.ac.jp/api/tree?action=browsing&more_ids=100', + 'https://test.sample.nii.ac.jp/api/tree?action=browsing&more_ids=100/201', + ] + ) + + @mock.patch('requests.get', side_effect=mock_requests_get_never_expanded) + def test_weko_get_indices_fails_when_weko_keeps_children_truncated(self, get_req_mock): + with assert_raises(ValueError): + self.conn.get_indices() + + @mock.patch('requests.get', side_effect=mock_requests_get) + def test_weko_get_indices_asks_for_the_given_language(self, get_req_mock): + self.conn.get_indices(accept_language='ja') + + get_req_mock.assert_called_once_with( + 'https://test.sample.nii.ac.jp/api/tree?action=browsing', + auth=(None, None), + headers={'Accept-Language': 'ja'}, + timeout=client.DEFAULT_TIMEOUT, + ) + @mock.patch('requests.get', side_effect=mock_requests_get) def test_weko_get_index_by_id(self, get_req_mock): index = self.conn.get_index_by_id(100) diff --git a/addons/weko/tests/test_serializer.py b/addons/weko/tests/test_serializer.py index 51d56e40930..6fbf5e5a1fe 100644 --- a/addons/weko/tests/test_serializer.py +++ b/addons/weko/tests/test_serializer.py @@ -82,6 +82,42 @@ def test_serialize_settings_authorized(self): assert_equal(serialized['ownerName'], self.user_settings.owner.fullname) assert_in('savedIndex', serialized) + def test_serialize_settings_names_indices_in_the_page_language(self): + def requests_get(url, **kwargs): + if kwargs.get('headers', {}).get('Accept-Language') == 'ja': + return utils.MockResponse([{'id': 100, 'name': 'サンプルインデックス'}], 200) + return utils.MockResponse([{'id': 100, 'name': 'Sample Index'}], 200) + + with mock.patch.object(type(self.node_settings), 'has_auth', return_value=True), \ + mock.patch.object(WEKOSerializer, 'credentials_are_valid', return_value=True), \ + mock.patch('requests.get', side_effect=requests_get), \ + self.app.app.test_request_context(headers={'Accept-Language': 'ja-JP'}): + serialized = self.ser.serialize_settings(self.node_settings, self.user, self.client) + + assert_equal(serialized['indices'], [{ + 'title': 'Sample Index', + 'localizedTitle': 'サンプルインデックス', + 'id': 100, + 'children': [], + }]) + + def test_serialize_settings_falls_back_to_the_default_name_and_warns(self): + def requests_get(url, **kwargs): + if kwargs.get('headers', {}).get('Accept-Language') == 'ja': + return utils.MockResponse([{'id': 100, 'name': 'サンプルインデックス'}], 200) + return utils.MockResponse([{'id': 100, 'name': 'Sample Index'}, {'id': 101, 'name': 'New Index'}], 200) + + with mock.patch.object(type(self.node_settings), 'has_auth', return_value=True), \ + mock.patch.object(WEKOSerializer, 'credentials_are_valid', return_value=True), \ + mock.patch('requests.get', side_effect=requests_get), \ + self.app.app.test_request_context(headers={'Accept-Language': 'ja'}), \ + self.assertLogs('addons.weko.serializer', level='WARNING') as logs: + serialized = self.ser.serialize_settings(self.node_settings, self.user, self.client) + + assert_equal([i['localizedTitle'] for i in serialized['indices']], ['サンプルインデックス', 'New Index']) + assert_equal(len(logs.records), 1) + assert_in('101', logs.records[0].getMessage()) + def test_serialize_settings_authorized_folder_is_set(self): pass diff --git a/addons/weko/tests/utils.py b/addons/weko/tests/utils.py index 076064d897a..f04a75ae656 100644 --- a/addons/weko/tests/utils.py +++ b/addons/weko/tests/utils.py @@ -11,9 +11,6 @@ 'id': 100, 'name': 'Sample Index', 'children': [ - { - 'id': 'more', - }, { 'id': 'dummy', } diff --git a/addons/wiki/models.py b/addons/wiki/models.py index e20f40dd730..f8c6a1bf75f 100644 --- a/addons/wiki/models.py +++ b/addons/wiki/models.py @@ -266,7 +266,16 @@ def get_for_node(self, node, name=None, id=None): deleted__isnull=True, node=node, ).order_by('created', 'id').first() - return WikiPage.load(id) + if id: + wiki_page = WikiPage.load(id) + # Restrict by node so wid cannot leak another project's page name. + if ( + wiki_page + and wiki_page.deleted is None + and wiki_page.node_id == node.id + ): + return wiki_page + return None def get_for_child_nodes(self, node, parent=None): if parent: diff --git a/addons/wiki/settings/defaults.py b/addons/wiki/settings/defaults.py index 9fadefc5f61..bbf8ed0f0de 100644 --- a/addons/wiki/settings/defaults.py +++ b/addons/wiki/settings/defaults.py @@ -8,6 +8,11 @@ SHAREJS_PORT = 7007 SHAREJS_URL = '{}:{}'.format(SHAREJS_HOST, SHAREJS_PORT) +Y_WEBSOCKET_SECRET = os.environ.get('Y_WEBSOCKET_SECRET', '') +Y_WEBSOCKET_JWT_ALGORITHM = 'HS256' +# Token validity for WebSocket connection (seconds). Not enforced after connection is established. +Y_WEBSOCKET_TOKEN_TTL = int(os.environ.get('Y_WEBSOCKET_TOKEN_TTL', 8 * 60 * 60)) + Y_WEBSOCKET_HOST = 'localhost' Y_WEBSOCKET_PORT = 1234 Y_WEBSOCKET_URL = '{}:{}'.format(Y_WEBSOCKET_HOST, Y_WEBSOCKET_PORT) diff --git a/addons/wiki/templates/edit.mako b/addons/wiki/templates/edit.mako index c4fd10d92da..6356b6f15e6 100644 --- a/addons/wiki/templates/edit.mako +++ b/addons/wiki/templates/edit.mako @@ -522,6 +522,7 @@ ${parent.javascript_bottom()} metadata: { registration: true, docId: ${ sharejs_uuid | sjson, n }, + yWebsocketToken: ${ y_websocket_token | sjson, n }, userId: ${user_id | sjson, n }, userName: ${ user_full_name | sjson, n }, userUrl: ${ user_url | sjson, n }, diff --git a/addons/wiki/tests/test_wiki.py b/addons/wiki/tests/test_wiki.py index 6daffd6dead..3be7918e3e3 100644 --- a/addons/wiki/tests/test_wiki.py +++ b/addons/wiki/tests/test_wiki.py @@ -35,7 +35,7 @@ from addons.wiki.utils import ( get_sharejs_uuid, generate_private_uuid, share_db, delete_share_doc, migrate_uuid, format_wiki_version, serialize_wiki_settings, serialize_wiki_widget, - check_file_object_in_node + check_file_object_in_node, generate_y_websocket_token ) from addons.wiki.views import WIKI_IMPORT_TASK_ALREADY_EXISTS from addons.wiki import tasks @@ -448,6 +448,17 @@ def test_wiki_id_url_get_returns_404(self): res = self.app.get(url, expect_errors=True) assert_equal(res.status_code, 404) + def test_wiki_id_url_get_returns_404_for_other_node_wiki(self): + other = ProjectFactory(is_public=True, creator=self.user) + other_page = WikiPage.objects.create_for_node( + other, 'SecretFromOther', 'content', Auth(self.user) + ) + url = self.project.web_url_for( + 'project_wiki_id_page', wid=other_page._primary_key, _guid=True + ) + res = self.app.get(url, expect_errors=True) + assert_equal(res.status_code, 404) + def test_home_is_capitalized_in_web_view(self): url = self.project.web_url_for('project_wiki_home', wid='home', _guid=True) res = self.app.get(url, auth=self.user.auth).follow(auth=self.user.auth) @@ -1861,11 +1872,21 @@ def test_project_wiki_validate_for_import(self, mock_check_file_object_in_node): mock_check_file_object_in_node.return_value = True dir_id = self.root_import_folder1._id url = self.project.api_url_for('project_wiki_validate_for_import', dir_id=dir_id) - res = self.app.get(url) + res = self.app.get(url, auth=self.user.auth) response_json = res.json task_id = response_json['taskId'] uuid_obj = uuid.UUID(task_id) assert uuid_obj + task = WikiImportTask.objects.get(task_id=task_id) + self.assertEqual(task.node, self.project) + self.assertEqual(task.status, WikiImportTask.STATUS_COMPLETED) + self.assertIsNotNone(task.process_end) + self.assertEqual( + WikiImportTask.objects.filter( + node=self.project, status=WikiImportTask.STATUS_RUNNING + ).count(), + 0, + ) @mock.patch('addons.wiki.utils.check_file_object_in_node') def test_project_wiki_validate_for_import_error(self, mock_check_file_object_in_node): @@ -1875,7 +1896,7 @@ def test_project_wiki_validate_for_import_error(self, mock_check_file_object_in_ )) dir_id = self.root_import_folder1._id url = self.project.api_url_for('project_wiki_validate_for_import', dir_id=dir_id) - res = self.app.get(url, expect_errors=True) + res = self.app.get(url, auth=self.user.auth, expect_errors=True) self.assertEqual(res.status_code, 400) self.assertEqual(res.json['message_short'], 'directory id does not exist') self.assertEqual(res.json['message_long'], 'directory id does not exist') @@ -1964,6 +1985,9 @@ def test_project_wiki_import(self, mock_check_file_object_in_node, mock_project_ response_json = res.json task_id = response_json['taskId'] uuid_obj = uuid.UUID(task_id) + task = WikiImportTask.objects.get(task_id=task_id) + self.assertEqual(task.node, self.project) + self.assertEqual(task.status, WikiImportTask.STATUS_RUNNING) @mock.patch('addons.wiki.utils.check_file_object_in_node') def test_project_wiki_import_error(self, mock_check_file_object_in_node): @@ -1983,6 +2007,43 @@ def test_project_wiki_import_with_no_admin_permission(self): res = self.app.post_json(url, { 'data': [{'test': 'test1'}] }, expect_errors=True) assert_equal(res.status_code, 401) + @mock.patch('addons.wiki.views.AsyncResult') + def test_project_get_task_result_with_admin_permission(self, mock_async_result): + mock_res = mock.MagicMock() + mock_res.ready.return_value = True + mock_res.get.return_value = {'canStartImport': True, 'data': []} + mock_async_result.return_value = mock_res + WikiImportTask.objects.create( + node=self.project, + task_id='task-id', + status=WikiImportTask.STATUS_COMPLETED, + creator=self.user, + ) + url = self.project.api_url_for('project_get_task_result', task_id='task-id') + res = self.app.get(url, auth=self.user.auth) + assert_equal(res.status_code, http_status.HTTP_200_OK) + assert_equal(res.json, {'canStartImport': True, 'data': []}) + + @mock.patch('addons.wiki.views.AsyncResult') + def test_project_get_task_result_rejects_task_for_other_node(self, mock_async_result): + other_project = ProjectFactory(creator=self.user) + WikiImportTask.objects.create( + node=other_project, + task_id='other-node-task-id', + status=WikiImportTask.STATUS_COMPLETED, + creator=self.user, + ) + url = self.project.api_url_for('project_get_task_result', task_id='other-node-task-id') + res = self.app.get(url, auth=self.user.auth, expect_errors=True) + assert_equal(res.status_code, http_status.HTTP_404_NOT_FOUND) + mock_async_result.assert_not_called() + + def test_project_get_task_result_rejects_unknown_task_id(self): + url = self.project.api_url_for('project_get_task_result', task_id='unknown-task-id') + res = self.app.get(url, auth=self.user.auth, expect_errors=True) + assert_equal(res.status_code, http_status.HTTP_404_NOT_FOUND) + + @mock.patch('celery.contrib.abortable.AbortableAsyncResult') def test_wiki_import_create_or_update_aborted(self, mock_task): mock_task.is_aborted.return_value = True @@ -3340,6 +3401,24 @@ def test_bulk_update_wiki_sort(self): self.assertEqual(result_wiki_child_page2, {'parent_id': wiki_page2_id, 'sort_order': 1}) self.assertEqual(result_wiki_child_page3, {'parent_id': wiki_child_page2_id, 'sort_order': 1}) + def test_bulk_update_wiki_sort_rejects_cross_node_parent(self): + other = ProjectFactory(is_public=True, creator=self.user) + other_page = WikiPage.objects.create_for_node( + other, 'other project wiki', 'content', self.consolidate_auth + ) + other_guid = other_page.guids.first()._id + parent_before = self.wiki_page1.parent_id + sort_id_list = [self.guid1] + sort_num_list = [1] + parent_wiki_id_list = [other_guid] + with self.assertRaises(HTTPError) as cm: + views._bulk_update_wiki_sort( + self.project, sort_id_list, sort_num_list, parent_wiki_id_list + ) + self.assertEqual(cm.exception.code, http_status.HTTP_400_BAD_REQUEST) + self.wiki_page1.reload() + self.assertEqual(self.wiki_page1.parent_id, parent_before) + def _wiki_sort_payload(self): return { 'sortedData': [ @@ -3380,3 +3459,112 @@ def test_project_update_wiki_page_sort(self): self.assertEqual(result_wiki_child_page1, {'parent_id': wiki_page2_id, 'sort_order': 1}) self.assertEqual(result_wiki_child_page2, {'parent_id': wiki_page2_id, 'sort_order': 2}) self.assertEqual(result_wiki_child_page3, {'parent_id': wiki_child_page2_id, 'sort_order': 1}) + + def test_project_update_wiki_page_sort_rejects_cross_node_parent(self): + other = ProjectFactory(is_public=True, creator=self.user) + other_page = WikiPage.objects.create_for_node( + other, 'other project wiki', 'content', self.consolidate_auth + ) + other_guid = other_page.guids.first()._id + parent_before = self.wiki_page1.parent_id + url = self.project.api_url_for('project_update_wiki_page_sort') + # Nest a local wiki under another project's wiki id as parent. + payload = { + 'sortedData': [ + { + 'name': 'other as parent', + 'id': other_guid, + 'sortOrder': 1, + 'children': [ + { + 'name': 'wiki page1', + 'id': self.guid1, + 'sortOrder': 1, + 'children': [], + 'fold': False, + } + ], + 'fold': False, + } + ] + } + res = self.app.post_json(url, payload, auth=self.user.auth, expect_errors=True) + assert_equal(res.status_code, http_status.HTTP_400_BAD_REQUEST) + self.wiki_page1.reload() + self.assertEqual(self.wiki_page1.parent_id, parent_before) + +@pytest.mark.enable_bookmark_creation +class TestYWebsocketToken(OsfTestCase): + + def setUp(self): + super(TestYWebsocketToken, self).setUp() + # Class-level @mock.patch is unreliable under pytest here; patch in setUp + # so generate_y_websocket_token sees a non-empty secret during the request. + self._y_websocket_secret = 'test-y-websocket-secret' + self._secret_patcher = mock.patch( + 'addons.wiki.settings.Y_WEBSOCKET_SECRET', + self._y_websocket_secret, + ) + self._secret_patcher.start() + self.addCleanup(self._secret_patcher.stop) + + self.user = AuthUserFactory() + self.project = ProjectFactory(is_public=True, creator=self.user) + self.wname = 'foo.bar' + self.wkey = to_mongo_key(self.wname) + + def test_token_generated_for_editor(self): + import jwt + from addons.wiki import settings as wiki_settings + + url = self.project.web_url_for('project_wiki_view', wname=self.wname) + res = self.app.get(url, auth=self.user.auth) + assert_equal(res.status_code, 200) + + self.project.reload() + body = res.body.decode() + sharejs_uuid = get_sharejs_uuid(self.project, self.wname) + assert_true(sharejs_uuid) + assert_in(sharejs_uuid, body) + + # edit.mako renders `yWebsocketToken: "..."`, not JSON `"yWebsocketToken": "..."`. + token_match = re.search(r'yWebsocketToken:\s*"([^"]+)"', body) + assert_true(token_match) + token = token_match.group(1) + assert_true(token) + + payload = jwt.decode( + token, + self._y_websocket_secret, + algorithms=[wiki_settings.Y_WEBSOCKET_JWT_ALGORITHM], + ) + assert_equal(payload['doc_id'], sharejs_uuid) + assert_equal(payload['sub'], self.user._id) + + def test_token_not_visible_without_write_permission(self): + WikiPage.objects.create_for_node(self.project, self.wname, 'some content', Auth(self.user)) + + url = self.project.web_url_for('project_wiki_view', wname=self.wname) + # Generate sharejs uuid / token path as an editor first (same pattern as TestWikiUuid). + res = self.app.get(url, auth=self.user.auth) + assert_equal(res.status_code, 200) + self.project.reload() + sharejs_uuid = get_sharejs_uuid(self.project, self.wname) + assert_true(sharejs_uuid) + + # Users without write permission should not receive uuid or a non-empty token. + res = self.app.get(url) + assert_equal(res.status_code, 200) + body = res.body.decode() + assert_not_in(sharejs_uuid, body) + token_match = re.search(r'yWebsocketToken:\s*"([^"]+)"', body) + assert_false(token_match) + + def test_generate_y_websocket_token_without_secret(self): + with mock.patch('addons.wiki.settings.Y_WEBSOCKET_SECRET', ''): + assert_equal(generate_y_websocket_token('doc-id', 'user-id'), '') + + def test_generate_y_websocket_token_without_user_id(self): + assert_equal(generate_y_websocket_token('doc-id', ''), '') + assert_equal(generate_y_websocket_token('doc-id', None), '') + diff --git a/addons/wiki/tests/test_wiki_import.py b/addons/wiki/tests/test_wiki_import.py index 73fe2c3a941..b57bf6545c4 100644 --- a/addons/wiki/tests/test_wiki_import.py +++ b/addons/wiki/tests/test_wiki_import.py @@ -1245,11 +1245,15 @@ def test_project_wiki_validate_for_import(self, mock_check_file_object_in_node): mock_check_file_object_in_node.return_value = True dir_id = self.root_import_folder1._id url = self.project.api_url_for('project_wiki_validate_for_import', dir_id=dir_id) - res = self.app.get(url) + res = self.app.get(url, auth=self.user.auth) response_json = res.json task_id = response_json['taskId'] uuid_obj = uuid.UUID(task_id) assert uuid_obj + task = WikiImportTask.objects.get(task_id=task_id) + assert_equal(task.node, self.project) + assert_equal(task.status, WikiImportTask.STATUS_COMPLETED) + assert_is_not_none(task.process_end) def test_project_wiki_validate_for_import_process(self): result = views.project_wiki_validate_for_import_process( @@ -1323,6 +1327,9 @@ def test_project_wiki_import(self, mock_check_file_object_in_node, mock_project_ task_id = response_json['taskId'] uuid_obj = uuid.UUID(task_id) assert_is_not_none(uuid_obj) + task = WikiImportTask.objects.get(task_id=task_id) + assert_equal(task.node, self.project) + assert_equal(task.status, WikiImportTask.STATUS_RUNNING) @mock.patch('addons.wiki.views._get_md_content_from_wb') @mock.patch('addons.wiki.views._get_or_create_wiki_folder') diff --git a/addons/wiki/utils.py b/addons/wiki/utils.py index 39af0660fb0..947cf31b834 100644 --- a/addons/wiki/utils.py +++ b/addons/wiki/utils.py @@ -4,10 +4,13 @@ import uuid import unicodedata import ssl +import datetime from future.moves.urllib.parse import quote +import jwt from pymongo import MongoClient import requests +from django.utils import timezone from bs4 import BeautifulSoup from django.apps import apps @@ -65,6 +68,32 @@ def get_sharejs_uuid(node, wname): )) if private_uuid else None +def generate_y_websocket_token(doc_id, user_id): + """ + Generate a signed JWT for y-websocket connection authorization. + + Includes ``sub`` (OSF user GUID) for connection traceability on the + y-websocket side. Returns an empty string when Y_WEBSOCKET_SECRET is not + configured or required claims are missing. + """ + secret = wiki_settings.Y_WEBSOCKET_SECRET + if not secret or not doc_id or not user_id: + return '' + + payload = { + 'doc_id': doc_id, + 'sub': user_id, + 'exp': timezone.now() + datetime.timedelta(seconds=wiki_settings.Y_WEBSOCKET_TOKEN_TTL), + } + token = jwt.encode( + payload, + secret, + algorithm=wiki_settings.Y_WEBSOCKET_JWT_ALGORITHM, + ) + if isinstance(token, bytes): + return token.decode() + return token + def delete_share_doc(node, wname): """Deletes share document and removes namespace from model.""" diff --git a/addons/wiki/views.py b/addons/wiki/views.py index a1051ba9110..bcf8b905095 100644 --- a/addons/wiki/views.py +++ b/addons/wiki/views.py @@ -93,6 +93,10 @@ message_short='Invalid request', message_long='The requested version of this wiki page does not exist.' )) +WIKI_INVALID_PARENT_ERROR = HTTPError(http_status.HTTP_400_BAD_REQUEST, data=dict( + message_short='Invalid parent wiki', + message_long='Parent wiki page does not exist on this project.', +)) WIKI_IMPORT_TASK_ALREADY_EXISTS = HTTPError(http_status.HTTP_400_BAD_REQUEST, data=dict( message_short='Running Task exists', @@ -325,10 +329,15 @@ def project_wiki_view(auth, wname, path=None, **kwargs): rendered_before_update = False markdown = '' + y_websocket_token = '' if can_edit: if wiki_key not in node.wiki_private_uuids: wiki_utils.generate_private_uuid(node, wiki_name) sharejs_uuid = wiki_utils.get_sharejs_uuid(node, wiki_name) + y_websocket_token = wiki_utils.generate_y_websocket_token( + sharejs_uuid, + auth.user._id, + ) else: if not wiki_page and wiki_key != 'home': raise WIKI_PAGE_NOT_FOUND_ERROR @@ -366,6 +375,7 @@ def project_wiki_view(auth, wname, path=None, **kwargs): 'sharejs_uuid': sharejs_uuid or '', 'sharejs_url': settings.SHAREJS_URL, 'y_websocket_url': settings.Y_WEBSOCKET_URL, + 'y_websocket_token': y_websocket_token, 'is_current': is_current, 'version_settings': version_settings, 'pages_current': _get_wiki_pages_latest(node), @@ -771,12 +781,40 @@ def serialize_component_wiki(node, auth): return component return None + +def _bind_wiki_task_to_node(task_id, node, auth, status, set_process_end=False): + """Record task_id as belonging to node for get_task_result authorization. + + Validate tasks must not use STATUS_RUNNING so they do not affect + "importing" UI, abort, or concurrent-import checks. + """ + defaults = { + 'node': node, + 'status': status, + 'creator': auth.user if auth else None, + } + if set_process_end: + defaults['process_end'] = timezone.make_naive(timezone.now(), timezone.utc) + WikiImportTask.objects.get_or_create(task_id=task_id, defaults=defaults) + + +def _wiki_task_belongs_to_node(task_id, node): + return WikiImportTask.objects.filter(task_id=task_id, node=node).exists() + + @must_be_valid_project -def project_wiki_validate_for_import(dir_id, node, **kwargs): +@must_have_permission(ADMIN) +@must_not_be_registration +@must_have_addon('wiki', 'node') +def project_wiki_validate_for_import(dir_id, auth, node, **kwargs): wiki_utils.check_file_object_in_node(dir_id, node) node_id = node.guids.first()._id task = tasks.run_project_wiki_validate_for_import.delay(dir_id, node_id) task_id = task.id + # Completed + process_end: ownership only; not treated as a running import. + _bind_wiki_task_to_node( + task_id, node, auth, WikiImportTask.STATUS_COMPLETED, set_process_end=True + ) return {'taskId': task_id} def project_wiki_validate_for_import_process(dir_id, node): @@ -904,11 +942,15 @@ def project_wiki_import(dir_id, auth, node, **kwargs): data_json = json.dumps(data) task = tasks.run_project_wiki_import.delay(data_json, dir_id, current_user_id, node_id) task_id = task.id + # Bind before returning so polling get_task_result does not race the worker. + _bind_wiki_task_to_node(task_id, node, auth, WikiImportTask.STATUS_RUNNING) + check_running_task(task_id, node) return {'taskId': task_id} def project_wiki_import_process(data, dir_id, task_id, auth, node): logger.info('----WIKI IMPORT DIRECTORY_ID: {}, PROJECT_NAME: {} ----'.format(dir_id, node.title)) - WikiImportTask.objects.create(node=node, task_id=task_id, status=WikiImportTask.STATUS_RUNNING, creator=auth.user) + # May already exist if project_wiki_import bound the task before the worker ran. + _bind_wiki_task_to_node(task_id, node, auth, WikiImportTask.STATUS_RUNNING) check_running_task(task_id, node) ret = [] wiki_id_list = [] @@ -1306,8 +1348,15 @@ def _create_import_error_list(wiki_infos, imported_list): return import_errors @must_be_valid_project +@must_have_permission(ADMIN) +@must_not_be_registration @must_have_addon('wiki', 'node') def project_get_task_result(task_id, node, **kwargs): + if not _wiki_task_belongs_to_node(task_id, node): + raise HTTPError(http_status.HTTP_404_NOT_FOUND, data=dict( + message_short='Not found', + message_long='Task not found.', + )) res = AsyncResult(task_id, app=celery_app) result = None if not res.ready(): @@ -1408,7 +1457,10 @@ def _bulk_update_wiki_sort(node, sort_id_list, sort_num_list, parent_wiki_id_lis parent_wiki_id = parent_wiki_id_list[idx] setattr(page, 'sort_order', sort_order_number) if parent_wiki_id is not None: - parent = WikiPage.objects.get(guids___id=parent_wiki_id) + # Same-node only; global guid lookup would allow cross-project parents. + parent = WikiPage.objects.get_for_node(node, id=parent_wiki_id) + if parent is None: + raise WIKI_INVALID_PARENT_ERROR setattr(page, 'parent', parent) else: setattr(page, 'parent', None) diff --git a/admin/base/urls.py b/admin/base/urls.py index c63c9d20fbd..d201a3e5b38 100644 --- a/admin/base/urls.py +++ b/admin/base/urls.py @@ -1,62 +1,84 @@ from django.conf.urls import include, url from django.contrib import admin -from admin.base.settings import ADMIN_BASE, DEBUG +from django.http import Http404 +from admin.base.settings import ADMIN_BASE, DEBUG, ENABLE_LOGIN_FORM from admin.base import views base_pattern = '^{}'.format(ADMIN_BASE) + +def admin_login_disabled(request): + # admin.site.urls provides its own password-based login (AdminSite.login), + # which must not exist while password-based login is disabled. + raise Http404 + +admin_urls = [ + url(r'^$', views.home, name='home'), +] + +if not ENABLE_LOGIN_FORM: + # must be registered before admin.site.urls, otherwise ^admin/ matches first + admin_urls.append(url(r'^admin/login/$', admin_login_disabled, name='admin_login_disabled')) + +admin_urls += [ + url(r'^admin/', admin.site.urls), + url(r'^asset_files/', include('admin.asset_files.urls', namespace='asset_files')), + url(r'^banners/', include('admin.banners.urls', namespace='banners')), + url(r'^brands/', include('admin.brands.urls', namespace='brands')), + url(r'^spam/', include('admin.spam.urls', namespace='spam')), + url(r'^institutions/', include('admin.institutions.urls', namespace='institutions')), + url(r'^entitlements/', include('admin.entitlements.urls', namespace='entitlements')), + url(r'^quota_recalc/', include('admin.quota_recalc.urls', namespace='quota_recalc')), + url(r'^preprint_providers/', include('admin.preprint_providers.urls', namespace='preprint_providers')), + url(r'^collection_providers/', include('admin.collection_providers.urls', namespace='collection_providers')), + url(r'^registration_providers/', include('admin.registration_providers.urls', namespace='registration_providers')), + url(r'^account/', include('admin.common_auth.urls', namespace='auth')), +] + +if ENABLE_LOGIN_FORM: + # password recovery/reset only makes sense while password-based login itself is enabled + admin_urls.append(url(r'^password/', include('password_reset.urls'))) + +admin_urls += [ + url(r'^nodes/', include('admin.nodes.urls', namespace='nodes')), + url(r'^preprints/', include('admin.preprints.urls', namespace='preprints')), + url(r'^subjects/', include('admin.subjects.urls', namespace='subjects')), + url(r'^users/', include('admin.users.urls', namespace='users')), + url(r'^user-emails/', include('admin.user_emails.urls', namespace='user-emails')), + url(r'^maintenance/', include('admin.maintenance.urls', namespace='maintenance')), + url(r'^meetings/', include('admin.meetings.urls', + namespace='meetings')), + url(r'^metrics/', include('admin.metrics.urls', + namespace='metrics')), + url(r'^desk/', include('admin.desk.urls', + namespace='desk')), + url(r'^osf_groups/', include('admin.osf_groups.urls', namespace='osf_groups')), + url(r'^management/', include('admin.management.urls', namespace='management')), + url(r'^announcement/', include('admin.rdm_announcement.urls', namespace='announcement')), + url(r'^addons/', include('admin.rdm_addons.urls', namespace='addons')), + url(r'^oauth/', include('admin.rdm_addons.oauth.urls', namespace='oauth')), + url(r'^statistics/', include('admin.rdm_statistics.urls', namespace='statistics')), + url(r'^timestampadd/', include('admin.rdm_timestampadd.urls', namespace='timestampadd')), + url(r'^keymanagement/', include('admin.rdm_keymanagement.urls', namespace='keymanagement')), + url(r'^timestampsettings/', include('admin.rdm_timestampsettings.urls', namespace='timestampsettings')), + url(r'^custom_storage_location/', include('admin.rdm_custom_storage_location.urls', namespace='custom_storage_location')), + url(r'^institutional_storage_quota_control/', include('admin.institutional_storage_quota_control.urls', + namespace='institutional_storage_quota_control')), + url(r'^metadata/', include('admin.rdm_metadata.urls', namespace='metadata')), + url(r'^user_identification_information/', + include('admin.user_identification_information.urls', namespace='user_identification_information')), + url(r'^user_identification_information_admin/', + include('admin.user_identification_information_admin.urls', namespace='user_identification_information_admin')), + url(r'^project_limit_number/', include('admin.project_limit_number.urls', namespace='project_limit_number')), + url(r'^rdm_workflow/', include('admin.rdm_workflow.urls', namespace='rdm_workflow')), + url(r'^loa/', include('admin.loa.urls', namespace='loa')), +] + urlpatterns = [ ### ADMIN ### url( base_pattern, - include([ - url(r'^$', views.home, name='home'), - url(r'^admin/', admin.site.urls), - url(r'^asset_files/', include('admin.asset_files.urls', namespace='asset_files')), - url(r'^banners/', include('admin.banners.urls', namespace='banners')), - url(r'^brands/', include('admin.brands.urls', namespace='brands')), - url(r'^spam/', include('admin.spam.urls', namespace='spam')), - url(r'^institutions/', include('admin.institutions.urls', namespace='institutions')), - url(r'^entitlements/', include('admin.entitlements.urls', namespace='entitlements')), - url(r'^quota_recalc/', include('admin.quota_recalc.urls', namespace='quota_recalc')), - url(r'^preprint_providers/', include('admin.preprint_providers.urls', namespace='preprint_providers')), - url(r'^collection_providers/', include('admin.collection_providers.urls', namespace='collection_providers')), - url(r'^registration_providers/', include('admin.registration_providers.urls', namespace='registration_providers')), - url(r'^account/', include('admin.common_auth.urls', namespace='auth')), - url(r'^password/', include('password_reset.urls')), - url(r'^nodes/', include('admin.nodes.urls', namespace='nodes')), - url(r'^preprints/', include('admin.preprints.urls', namespace='preprints')), - url(r'^subjects/', include('admin.subjects.urls', namespace='subjects')), - url(r'^users/', include('admin.users.urls', namespace='users')), - url(r'^user-emails/', include('admin.user_emails.urls', namespace='user-emails')), - url(r'^maintenance/', include('admin.maintenance.urls', namespace='maintenance')), - url(r'^meetings/', include('admin.meetings.urls', - namespace='meetings')), - url(r'^metrics/', include('admin.metrics.urls', - namespace='metrics')), - url(r'^desk/', include('admin.desk.urls', - namespace='desk')), - url(r'^osf_groups/', include('admin.osf_groups.urls', namespace='osf_groups')), - url(r'^management/', include('admin.management.urls', namespace='management')), - url(r'^announcement/', include('admin.rdm_announcement.urls', namespace='announcement')), - url(r'^addons/', include('admin.rdm_addons.urls', namespace='addons')), - url(r'^oauth/', include('admin.rdm_addons.oauth.urls', namespace='oauth')), - url(r'^statistics/', include('admin.rdm_statistics.urls', namespace='statistics')), - url(r'^timestampadd/', include('admin.rdm_timestampadd.urls', namespace='timestampadd')), - url(r'^keymanagement/', include('admin.rdm_keymanagement.urls', namespace='keymanagement')), - url(r'^timestampsettings/', include('admin.rdm_timestampsettings.urls', namespace='timestampsettings')), - url(r'^custom_storage_location/', include('admin.rdm_custom_storage_location.urls', namespace='custom_storage_location')), - url(r'^institutional_storage_quota_control/', include('admin.institutional_storage_quota_control.urls', - namespace='institutional_storage_quota_control')), - url(r'^metadata/', include('admin.rdm_metadata.urls', namespace='metadata')), - url(r'^user_identification_information/', - include('admin.user_identification_information.urls', namespace='user_identification_information')), - url(r'^user_identification_information_admin/', - include('admin.user_identification_information_admin.urls', namespace='user_identification_information_admin')), - url(r'^project_limit_number/', include('admin.project_limit_number.urls', namespace='project_limit_number')), - url(r'^rdm_workflow/', include('admin.rdm_workflow.urls', namespace='rdm_workflow')), - url(r'^loa/', include('admin.loa.urls', namespace='loa')), - ]), + include(admin_urls), ), ] diff --git a/admin/base/utils.py b/admin/base/utils.py index 6ee8780be71..ff86f9f7900 100644 --- a/admin/base/utils.py +++ b/admin/base/utils.py @@ -1,6 +1,8 @@ """ Utility functions and classes """ +from functools import wraps + from osf.models import Subject, NodeLicense, Brand from django.core.exceptions import ValidationError, PermissionDenied @@ -35,6 +37,29 @@ def osf_staff_check(user): return user.is_authenticated and user.is_staff +def osf_superuser_check(user): + """An inactive or unregistered superuser is not an Integrated Admin.""" + return bool( + user.is_authenticated and + user.is_active and + user.is_registered and + user.is_superuser + ) + + +def superuser_required(view_func): + """Function-view decorator that returns 403 (PermissionDenied) instead of + redirecting to login, unlike django.contrib.auth.decorators.user_passes_test.""" + + @wraps(view_func) + def wrapped_view(request, *args, **kwargs): + if not osf_superuser_check(request.user): + raise PermissionDenied + return view_func(request, *args, **kwargs) + + return wrapped_view + + def get_subject_rules(subjects_selected): """ Take a list of subjects, and parse them into rules consistent with preprpint provider diff --git a/admin/collection_providers/views.py b/admin/collection_providers/views.py index cafb9476f9e..4cfd366d990 100644 --- a/admin/collection_providers/views.py +++ b/admin/collection_providers/views.py @@ -4,7 +4,7 @@ from django.core import serializers from django.core.exceptions import ValidationError from django.urls import reverse_lazy -from django.shortcuts import redirect +from django.shortcuts import redirect, get_object_or_404 from django.views.generic import View, CreateView, ListView, DetailView, UpdateView, DeleteView, TemplateView from django.contrib import messages from django.contrib.auth.mixins import PermissionRequiredMixin @@ -228,12 +228,14 @@ def get_object(self, queryset=None): return CollectionProvider.objects.get(id=self.kwargs['collection_provider_id']) -class CannotDeleteProvider(TemplateView): +class CannotDeleteProvider(PermissionRequiredMixin, TemplateView): + permission_required = 'osf.delete_collectionprovider' + raise_exception = True template_name = 'collection_providers/cannot_delete.html' def get_context_data(self, **kwargs): context = super(CannotDeleteProvider, self).get_context_data(**kwargs) - context['provider'] = CollectionProvider.objects.get(id=self.kwargs['collection_provider_id']) + context['provider'] = get_object_or_404(CollectionProvider, id=self.kwargs['collection_provider_id']) return context diff --git a/admin/common_auth/views.py b/admin/common_auth/views.py index d2dc3993090..dd03b30dedb 100644 --- a/admin/common_auth/views.py +++ b/admin/common_auth/views.py @@ -39,10 +39,12 @@ def dispatch(self, request, *args, **kwargs): return super(LoginView, self).dispatch(request, *args, **kwargs) def form_valid(self, form): - user = authenticate( - username=form.cleaned_data.get('email').strip(), - password=form.cleaned_data.get('password').strip() - ) + user = None + if ENABLE_LOGIN_FORM: + user = authenticate( + username=form.cleaned_data.get('email').strip(), + password=form.cleaned_data.get('password').strip() + ) if user is not None: login(self.request, user) else: diff --git a/admin/management/views.py b/admin/management/views.py index fffd1bbef4f..b1522991ad2 100644 --- a/admin/management/views.py +++ b/admin/management/views.py @@ -1,15 +1,29 @@ +from django.contrib.auth.mixins import UserPassesTestMixin from django.views.generic import TemplateView, View from osf.management.commands.manage_switch_flags import manage_waffle from django.core.urlresolvers import reverse from django.shortcuts import redirect -class ManagementCommands(TemplateView): +from admin.rdm.utils import RdmPermissionMixin + +class ManagementCommands(RdmPermissionMixin, UserPassesTestMixin, TemplateView): """ Basic form to trigger various management commands """ template_name = 'management/commands.html' object_type = 'management' + raise_exception = True + + def test_func(self): + """check user permissions""" + return self.is_super_admin + + +class WaffleFlag(RdmPermissionMixin, UserPassesTestMixin, View): + raise_exception = True -class WaffleFlag(View): + def test_func(self): + """check user permissions""" + return self.is_super_admin def post(self, request, *args, **kwargs): manage_waffle() diff --git a/admin/preprint_providers/views.py b/admin/preprint_providers/views.py index 159a0a409f5..8d2ecbf6d29 100644 --- a/admin/preprint_providers/views.py +++ b/admin/preprint_providers/views.py @@ -14,7 +14,7 @@ from django.contrib import messages from django.contrib.auth.mixins import PermissionRequiredMixin from django.forms.models import model_to_dict -from django.shortcuts import redirect, render +from django.shortcuts import redirect, render, get_object_or_404 from admin.base import settings from admin.base.forms import ImportFileForm @@ -319,12 +319,14 @@ def get_object(self, queryset=None): return PreprintProvider.objects.get(id=self.kwargs['preprint_provider_id']) -class CannotDeleteProvider(TemplateView): +class CannotDeleteProvider(PermissionRequiredMixin, TemplateView): + permission_required = 'osf.delete_preprintprovider' + raise_exception = True template_name = 'preprint_providers/cannot_delete.html' def get_context_data(self, **kwargs): context = super(CannotDeleteProvider, self).get_context_data(**kwargs) - context['provider'] = PreprintProvider.objects.get(id=self.kwargs['preprint_provider_id']) + context['provider'] = get_object_or_404(PreprintProvider, id=self.kwargs['preprint_provider_id']) return context diff --git a/admin/quota_recalc/views.py b/admin/quota_recalc/views.py index 29484c3d559..540e1acef0d 100644 --- a/admin/quota_recalc/views.py +++ b/admin/quota_recalc/views.py @@ -1,6 +1,7 @@ from django.http import JsonResponse from django.db import transaction, IntegrityError +from admin.base.utils import superuser_required from addons.osfstorage.models import Region from api.base import settings as api_settings from osf.models import OSFUser, UserQuota, Node @@ -60,6 +61,7 @@ def calculate_quota(user): user_quota.used = used user_quota.save() +@superuser_required def all_users(request, **kwargs): c = 0 for osf_user in OSFUser.objects.exclude(deleted__isnull=False): @@ -70,6 +72,7 @@ def all_users(request, **kwargs): 'message': str(c) + ' users\' quota successfully recalculated!' }) +@superuser_required def user(request, guid, **kwargs): user = OSFUser.load(guid) if user is None: diff --git a/admin/rdm_custom_storage_location/cron.sh b/admin/rdm_custom_storage_location/cron.sh index 10cb90c4250..81c65448528 100644 --- a/admin/rdm_custom_storage_location/cron.sh +++ b/admin/rdm_custom_storage_location/cron.sh @@ -1 +1 @@ -*/30 * * * * curl http://localhost:8001/custom_storage_location/external_acc_update/d610ef95f0b0f5868f13919b8ed64070b9acb9c19b8da9f2c514ed938203ec3e236c9cad4f4146bdf22b4e79cf0d92f6d4f4c996d236c6b0ee79a1336b26afb7/ | jq -c . +*/30 * * * * TOKEN=$(cd /code && python3 manage.py generate_signed_cron_token | tail -n1) && [ -n "$TOKEN" ] && curl "http://localhost:8001/custom_storage_location/external_acc_update/$TOKEN/" | jq -c . diff --git a/admin/rdm_custom_storage_location/urls.py b/admin/rdm_custom_storage_location/urls.py index 6a62d690294..c51f34c148e 100644 --- a/admin/rdm_custom_storage_location/urls.py +++ b/admin/rdm_custom_storage_location/urls.py @@ -3,7 +3,7 @@ urlpatterns = [ - url(r'^external_acc_update/(?P-?\w+)/$', views.external_acc_update, name='external_acc_update'), + url(r'^external_acc_update/(?P\d+)/(?P[0-9a-fA-F]+)/$', views.external_acc_update, name='external_acc_update'), url(r'^institutional_storage/$', views.InstitutionalStorageView.as_view(), name='institutional_storage'), url(r'^institutional_storage/institutions/$', views.InstitutionalStorageListView.as_view(), name='institutional_storage_institutions'), url(r'^institutional_storage/institutions/(?P[0-9]+)/$', views.InstitutionalStorageView.as_view(), name='institutional_storage_list'), diff --git a/admin/rdm_custom_storage_location/utils.py b/admin/rdm_custom_storage_location/utils.py index df350ed2ae1..ae6ab5209d6 100644 --- a/admin/rdm_custom_storage_location/utils.py +++ b/admin/rdm_custom_storage_location/utils.py @@ -261,7 +261,10 @@ def test_s3_connection(access_key, secret_key, bucket): s3_response = { 'id': user_info.id, 'display_name': user_info.display_name, - 'Owner': user_info.Owner, + 'Owner': { + 'ID': user_info.id, + 'DisplayName': user_info.display_name, + }, } return ({ diff --git a/admin/rdm_custom_storage_location/views.py b/admin/rdm_custom_storage_location/views.py index dff08d2b3ec..33dd5824844 100644 --- a/admin/rdm_custom_storage_location/views.py +++ b/admin/rdm_custom_storage_location/views.py @@ -5,7 +5,6 @@ from django.views.generic import TemplateView, View, ListView import codecs import json -import hashlib from rest_framework import status as http_status from mimetypes import MimeTypes import os @@ -19,13 +18,13 @@ from osf.models import Institution, OSFUser from osf.models.external import ExternalAccountTemporary from scripts import refresh_addon_tokens +from framework.auth import cron_signed_url from website import settings as osf_settings from distutils.util import strtobool from admin.base import settings logger = logging.getLogger(__name__) -SITE_KEY = 'rdm_custom_storage_location' INSTITUTION_NOT_FOUND_MESSAGE = 'Institution does not exist' class InstitutionalStorageBaseView(RdmPermissionMixin, UserPassesTestMixin): @@ -522,8 +521,8 @@ def post(self, request, *args, **kwargs): 'message': 'Garbage data removed!!' }, status=http_status.HTTP_200_OK) -def external_acc_update(request, access_token): - if hashlib.sha512(SITE_KEY.encode('utf-8')).hexdigest() != access_token.lower(): +def external_acc_update(request, ts, signature): + if not cron_signed_url.verify_signed_params(ts, signature): return HttpResponse( json.dumps({'state': 'fail', 'error': 'access forbidden'}), content_type='application/json', diff --git a/admin/rdm_statistics/cron.sh b/admin/rdm_statistics/cron.sh index 62a80709268..a3d434e7808 100644 --- a/admin/rdm_statistics/cron.sh +++ b/admin/rdm_statistics/cron.sh @@ -1 +1 @@ -0 2 * * 1 curl http://localhost:8001/statistics/gather/2A85563B2B0F7D3168199F475365F57DA1D56E4BB2CE2B7044EB058AE5E287637E7C636A772682D92C8D6B1830B9A97C5A5DC3DE7016C60BDE4BAA7CC3B38AEB/ | jq -c . +0 2 * * 1 TOKEN=$(cd /code && python3 manage.py generate_signed_cron_token | tail -n1) && [ -n "$TOKEN" ] && curl "http://localhost:8001/statistics/gather/$TOKEN/" | jq -c . diff --git a/admin/rdm_statistics/urls.py b/admin/rdm_statistics/urls.py index 5775b6bf71e..443de572baf 100644 --- a/admin/rdm_statistics/urls.py +++ b/admin/rdm_statistics/urls.py @@ -8,7 +8,7 @@ url(r'^index$', views.IndexView.as_view(), name='index'), url(r'^(?P-?[0-9]+)/graph/(?P\w+)_(?P\w+)\.(\w+)$', views.ImageView.as_view(), name='graph'), - url(r'^gather/(?P-?\w+)/$', views.GatherView.as_view(), name='gather'), + url(r'^gather/(?P\d+)/(?P[0-9a-fA-F]+)/$', views.GatherView.as_view(), name='gather'), url(r'^report/(?P-?[0-9]+)/$', views.create_pdf, name='report'), url(r'^csv/(?P-?[0-9]+)/$', views.create_csv, name='csv'), url(r'^mail/(?P-?[0-9]+)/$', views.SendView.as_view(), name='mail'), diff --git a/admin/rdm_statistics/views.py b/admin/rdm_statistics/views.py index d4522cf1a9c..20f8dc23017 100644 --- a/admin/rdm_statistics/views.py +++ b/admin/rdm_statistics/views.py @@ -12,7 +12,6 @@ import csv import pandas as pd import numpy as np -import hashlib from django.apps import apps from django.views.generic import TemplateView, View @@ -41,8 +40,10 @@ import seaborn as sns import pdfkit from admin.base import settings +from admin.base.utils import superuser_required from admin.rdm.utils import RdmPermissionMixin, get_dummy_institution from admin.rdm_addons import utils +from framework.auth import cron_signed_url import logging logger = logging.getLogger(__name__) @@ -51,7 +52,6 @@ STATISTICS_IMAGE_HEIGHT = 4 RECURSIVE_LIMIT = 10000 WB_MAX_RETRY = 3 -SITE_KEY = 'rdm_statistics' class InstitutionListViewStat(RdmPermissionMixin, UserPassesTestMixin, TemplateView): """institlutions list view for statistics""" @@ -113,8 +113,6 @@ def get_context_data(self, **kwargs): ctx['current_date'] = current_date ctx['user'] = user ctx['provider_data_array'] = provider_data_array - digest = hashlib.sha512(SITE_KEY.encode('utf-8')).hexdigest() - ctx['token'] = digest.upper() return ctx @@ -447,9 +445,10 @@ class GatherView(TemplateView): raise_exception = True def get(self, request, *args, **kwargs): - # simple authentication - access_token = self.kwargs.get('access_token') - if not simple_auth(access_token): + # signed-URL authentication + ts = self.kwargs.get('ts') + signature = self.kwargs.get('signature') + if not cron_signed_url.verify_signed_params(ts, signature): response_hash = {'state': 'fail', 'error': 'access forbidden'} response_json = json.dumps(response_hash) response = HttpResponse(response_json, content_type='application/json') @@ -484,7 +483,7 @@ def get(self, request, *args, **kwargs): response_json = json.dumps(self.stat_list) response = HttpResponse(response_json, content_type='application/json') # statistics mail send - send_stat_mail(request) + send_stat_mail_core(request) except Exception as err: response_hash = {'state': 'fail', 'error': str(err)} response_json = json.dumps(response_hash) @@ -589,15 +588,15 @@ def count_project_files(self, node_id, provider, path, cookies): pass self.count_project_files(provider=provider, node_id=node_id, path='/' + path, cookies=cookies) -def simple_auth(access_token): - digest = hashlib.sha512(SITE_KEY.encode('utf-8')).hexdigest() - if digest == access_token.lower(): - return True - else: - return False -def send_stat_mail(request, **kwargs): - """send statistics information email""" +def send_stat_mail_core(request, **kwargs): + """Send statistics information email. + + No authorization check here by design - this is the shared logic called + both by the send_stat_mail view (protected below) and internally by + GatherView.get(), which runs unauthenticated over localhost from + cron and is authorized separately via its own access token. + """ current_date = get_current_date() all_institutions = Institution.objects.order_by('id').all() all_staff_users = OSFUser.objects.filter(is_staff=True) @@ -630,6 +629,11 @@ def send_stat_mail(request, **kwargs): response = HttpResponse(response_json, content_type='application/json') return response +@superuser_required +def send_stat_mail(request, **kwargs): + """URL-exposed view for /statistics/test/mail/ - Integrated Admin only.""" + return send_stat_mail_core(request, **kwargs) + def send_error_mail(err): """send error email""" current_date = get_current_date() @@ -776,14 +780,18 @@ def approximate_size(size, a_kilobyte_is_1024_bytes=True): ### views or funcs for development and test ############################################ -class IndexView(TemplateView): +class IndexView(RdmPermissionMixin, UserPassesTestMixin, TemplateView): """index view of statistics module.""" template_name = 'rdm_statistics/index.html' raise_exception = True + def test_func(self): + """check user permissions""" + return self.is_authenticated and (self.is_super_admin or self.is_admin) + def find_bookmark_collection(self, user): collection = apps.get_model('osf.Collection') - return collection.objects.get(creator=user, is_deleted=False, is_bookmark_collection=True) + return collection.objects.get(creator=user, deleted__isnull=True, is_bookmark_collection=True) def get(self, request, *args, **kwargs): user = self.request.user diff --git a/admin/registration_providers/views.py b/admin/registration_providers/views.py index 2466bfae63a..c576ed7db16 100644 --- a/admin/registration_providers/views.py +++ b/admin/registration_providers/views.py @@ -6,7 +6,7 @@ from django.core.exceptions import ValidationError from django.core.management import call_command from django.urls import reverse_lazy -from django.shortcuts import redirect +from django.shortcuts import redirect, get_object_or_404 from django.views.generic import View, CreateView, ListView, DetailView, UpdateView, DeleteView, TemplateView from django.contrib.auth.mixins import PermissionRequiredMixin from django.contrib import messages @@ -204,12 +204,14 @@ def get_context_data(self, *args, **kwargs): return super(DeleteRegistrationProvider, self).get_context_data(*args, **kwargs) -class CannotDeleteProvider(TemplateView): +class CannotDeleteProvider(PermissionRequiredMixin, TemplateView): + permission_required = 'osf.delete_registrationprovider' + raise_exception = True template_name = 'registration_providers/cannot_delete.html' def get_context_data(self, **kwargs): context = super(CannotDeleteProvider, self).get_context_data(**kwargs) - context['provider'] = RegistrationProvider.objects.get(id=self.kwargs['registration_provider_id']) + context['provider'] = get_object_or_404(RegistrationProvider, id=self.kwargs['registration_provider_id']) return context @@ -376,7 +378,7 @@ def get(self, request, *args, **kwargs): return redirect(reverse_lazy('registration_providers:detail', kwargs={'registration_provider_id': provider.id})) -class ChangeSchema(TemplateView): +class ChangeSchema(PermissionRequiredMixin, TemplateView): permission_required = 'osf.change_registrationprovider' template_name = 'registration_providers/change_schema.html' diff --git a/admin_tests/base/test_urls.py b/admin_tests/base/test_urls.py new file mode 100644 index 00000000000..c6c4ff60aa5 --- /dev/null +++ b/admin_tests/base/test_urls.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +import mock +from django.urls import clear_url_caches, resolve +from django.http import Http404 +from django.test import RequestFactory +from django.utils.six.moves import reload_module +from nose import tools as nt + +from admin.base import urls as admin_urls_module +from tests.base import AdminTestCase + + +class TestAdminLoginUrl(AdminTestCase): + """/admin/login/ (AdminSite.login from admin.site.urls) is a second entry point + of password-based authentication, so it must follow ENABLE_LOGIN_FORM.""" + + def _reload_urls(self): + reload_module(admin_urls_module) + clear_url_caches() + + def _with_login_form(self, enabled): + patcher = mock.patch('admin.base.settings.ENABLE_LOGIN_FORM', enabled) + patcher.start() + self.addCleanup(self._restore_urls) + self.addCleanup(patcher.stop) + self._reload_urls() + + def _restore_urls(self): + # cleanups run LIFO: patcher.stop() has already run, so this reloads with the real setting + reload_module(admin_urls_module) + clear_url_caches() + + def _call_login_view(self, request): + match = resolve('/admin/login/') + return match.func(request, *match.args, **match.kwargs) + + def test_login_is_404_for_get_when_login_form_disabled(self): + self._with_login_form(False) + request = RequestFactory().get('/admin/login/') + with nt.assert_raises(Http404): + self._call_login_view(request) + + def test_login_is_404_for_post_when_login_form_disabled(self): + self._with_login_form(False) + request = RequestFactory().post('/admin/login/', {'username': 'user@example.com', 'password': 'secret'}) + with nt.assert_raises(Http404): + self._call_login_view(request) + + def test_login_route_is_disabled_view_when_login_form_disabled(self): + self._with_login_form(False) + nt.assert_equal(resolve('/admin/login/').url_name, 'admin_login_disabled') + + def test_login_route_is_admin_site_login_when_login_form_enabled(self): + self._with_login_form(True) + nt.assert_equal(resolve('/admin/login/').view_name, 'admin:login') + + def test_admin_site_urls_other(self): + self._with_login_form(False) + nt.assert_equal(resolve('/admin/').view_name, 'admin:index') diff --git a/admin_tests/base/test_utils.py b/admin_tests/base/test_utils.py index 37738383542..910b4f02e63 100644 --- a/admin_tests/base/test_utils.py +++ b/admin_tests/base/test_utils.py @@ -17,7 +17,7 @@ from osf.models import Subject, OSFUser, Collection from osf.models.provider import rules_to_subjects -from admin.base.utils import get_subject_rules, change_embargo_date +from admin.base.utils import get_subject_rules, change_embargo_date, osf_superuser_check from osf.admin import OSFUserAdmin @@ -255,3 +255,39 @@ def test_from_json(self): def test_from_json__file_not_found(self): with pytest.raises(Exception): from_json('file-info-schema2.json') + + +class TestOsfSuperuserCheck(AdminTestCase): + """osf_superuser_check must have the same condition as RdmPermissionMixin.is_super_admin.""" + + def setUp(self): + super(TestOsfSuperuserCheck, self).setUp() + self.user = UserFactory() + self.user.is_superuser = True + self.user.save() + + def _update(self, **fields): + # queryset.update() skips OSFUser.save(), which would otherwise recompute is_active + OSFUser.objects.filter(pk=self.user.pk).update(**fields) + self.user.refresh_from_db() + + def test_valid_superuser(self): + assert_true(osf_superuser_check(self.user)) + + def test_non_superuser(self): + self._update(is_superuser=False) + assert_false(osf_superuser_check(self.user)) + + def test_inactive_superuser(self): + self._update(is_active=False) + assert_true(self.user.is_superuser) + assert_false(osf_superuser_check(self.user)) + + def test_unregistered_superuser(self): + self._update(is_registered=False) + assert_true(self.user.is_superuser) + assert_false(osf_superuser_check(self.user)) + + def test_anonymous(self): + from django.contrib.auth.models import AnonymousUser + assert_false(osf_superuser_check(AnonymousUser())) diff --git a/admin_tests/collection_providers/test_views.py b/admin_tests/collection_providers/test_views.py index ef054a2778f..1dd19e84801 100644 --- a/admin_tests/collection_providers/test_views.py +++ b/admin_tests/collection_providers/test_views.py @@ -1,6 +1,9 @@ import pytest from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser, Permission +from django.core.exceptions import PermissionDenied +from django.http import Http404 from osf_tests.factories import ( AuthUserFactory, @@ -93,3 +96,45 @@ def view(self, req, provider): view = setup_view(view, req) view.kwargs = {'collection_provider_id': provider.id} return view + + +@pytest.mark.urls('admin.base.urls') +class TestCannotDeleteProviderPermission: + """CannotDeleteProvider must be restricted to users with 'osf.delete_collectionprovider'.""" + + @pytest.fixture() + def provider(self): + return CollectionProviderFactory() + + def test_denied_for_general_user(self, req, provider): + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, collection_provider_id=provider.id) + + def test_denied_for_anonymous(self, provider): + req = RequestFactory().get('/fake_path') + req.user = AnonymousUser() + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, collection_provider_id=provider.id) + + def test_allowed_for_user_with_permission(self, req, user, provider): + permission = Permission.objects.get(codename='delete_collectionprovider') + user.user_permissions.add(permission) + user.save() + + res = views.CannotDeleteProvider.as_view()(req, collection_provider_id=provider.id) + assert res.status_code == 200 + + def test_allowed_for_superuser(self, req, user, provider): + user.is_superuser = True + user.save() + + res = views.CannotDeleteProvider.as_view()(req, collection_provider_id=provider.id) + assert res.status_code == 200 + + def test_404_for_missing_provider(self, req, user): + permission = Permission.objects.get(codename='delete_collectionprovider') + user.user_permissions.add(permission) + user.save() + + with pytest.raises(Http404): + views.CannotDeleteProvider.as_view()(req, collection_provider_id=999999) diff --git a/admin_tests/common_auth/test_views.py b/admin_tests/common_auth/test_views.py index 5bc23647714..89bc21614f7 100644 --- a/admin_tests/common_auth/test_views.py +++ b/admin_tests/common_auth/test_views.py @@ -5,14 +5,15 @@ from django.http import Http404 from django.urls import reverse from django.contrib.auth import REDIRECT_FIELD_NAME +from django.contrib.messages.storage.fallback import FallbackStorage from tests.base import AdminTestCase from osf_tests.factories import AuthUserFactory, InstitutionFactory from admin_tests.utilities import setup_form_view, setup_view from osf.models.user import OSFUser -from admin.common_auth.views import RegisterUser, ShibLoginView -from admin.common_auth.forms import UserRegistrationForm +from admin.common_auth.views import RegisterUser, ShibLoginView, LoginView +from admin.common_auth.forms import UserRegistrationForm, LoginForm class TestRegisterUser(AdminTestCase): @@ -356,3 +357,47 @@ def test_new_user_missing_displayname_header_raises_key_error(self, mock_use_epp view = setup_view(ShibLoginView(), request) with nt.assert_raises(KeyError): view.dispatch(request) + + +class TestLoginViewFormValid(AdminTestCase): + """LoginView.form_valid must skip authenticate() entirely when + ENABLE_LOGIN_FORM is False, so password-based login stays disabled even via + a direct POST to /account/login/.""" + + def setUp(self): + super(TestLoginViewFormValid, self).setUp() + self.view = LoginView() + self.request = RequestFactory().post('/account/login/') + # django.contrib.messages has a bug which effects unittests + # more info here -> https://code.djangoproject.com/ticket/17971 + setattr(self.request, 'session', 'session') + setattr(self.request, '_messages', FallbackStorage(self.request)) + self.form = LoginForm(data={'email': 'user@example.com', 'password': 'secret'}) + nt.assert_true(self.form.is_valid()) + self.view = setup_form_view(self.view, self.request, self.form) + + @mock.patch('admin.common_auth.views.ENABLE_LOGIN_FORM', False) + @mock.patch('admin.common_auth.views.authenticate') + def test_authenticate_skipped_when_login_form_disabled(self, mock_authenticate): + response = self.view.form_valid(self.form) + nt.assert_false(mock_authenticate.called) + nt.assert_equal(response.status_code, 302) + nt.assert_true(response.url.startswith('/account/login')) + + @mock.patch('admin.common_auth.views.ENABLE_LOGIN_FORM', True) + @mock.patch('admin.common_auth.views.authenticate', return_value=None) + def test_authenticate_called_and_denied_when_login_form_enabled(self, mock_authenticate): + response = self.view.form_valid(self.form) + nt.assert_true(mock_authenticate.called) + nt.assert_equal(response.status_code, 302) + nt.assert_true(response.url.startswith('/account/login')) + + @mock.patch('admin.common_auth.views.ENABLE_LOGIN_FORM', True) + @mock.patch('admin.common_auth.views.login') + @mock.patch('admin.common_auth.views.authenticate') + def test_authenticate_called_and_login_succeeds_when_login_form_enabled(self, mock_authenticate, mock_login): + mock_authenticate.return_value = AuthUserFactory() + response = self.view.form_valid(self.form) + nt.assert_true(mock_authenticate.called) + nt.assert_true(mock_login.called) + nt.assert_equal(response.status_code, 302) diff --git a/admin_tests/management/__init__.py b/admin_tests/management/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/admin_tests/management/test_views.py b/admin_tests/management/test_views.py new file mode 100644 index 00000000000..8e93b66af0f --- /dev/null +++ b/admin_tests/management/test_views.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +from nose import tools as nt +import mock +from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser +from django.core.exceptions import PermissionDenied + +from admin.management import views +from osf_tests.factories import AuthUserFactory +from tests.base import AdminTestCase + + +class TestManagementCommandsPermission(AdminTestCase): + """ManagementCommands must be restricted to Integrated Admin (is_super_admin).""" + + def setUp(self): + super(TestManagementCommandsPermission, self).setUp() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + self.general_user = AuthUserFactory() + + def test_get_denied_for_general_user(self): + request = RequestFactory().get('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.ManagementCommands.as_view()(request) + + def test_get_denied_for_anonymous(self): + request = RequestFactory().get('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.ManagementCommands.as_view()(request) + + def test_get_allowed_for_superuser(self): + request = RequestFactory().get('/fake_path') + request.user = self.superuser + response = views.ManagementCommands.as_view()(request) + nt.assert_equal(response.status_code, 200) + + +class TestWaffleFlagPermission(AdminTestCase): + """WaffleFlag must be restricted to Integrated Admin (is_super_admin).""" + + def setUp(self): + super(TestWaffleFlagPermission, self).setUp() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + self.general_user = AuthUserFactory() + + def test_post_denied_for_general_user(self): + request = RequestFactory().post('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.WaffleFlag.as_view()(request) + + def test_post_denied_for_anonymous(self): + request = RequestFactory().post('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.WaffleFlag.as_view()(request) + + @mock.patch('admin.management.views.manage_waffle') + def test_post_allowed_for_superuser(self, mock_manage_waffle): + request = RequestFactory().post('/fake_path') + request.user = self.superuser + response = views.WaffleFlag.as_view()(request) + nt.assert_equal(response.status_code, 302) + nt.assert_true(mock_manage_waffle.called) diff --git a/admin_tests/preprint_providers/test_views.py b/admin_tests/preprint_providers/test_views.py index 66403b89536..204107770a3 100644 --- a/admin_tests/preprint_providers/test_views.py +++ b/admin_tests/preprint_providers/test_views.py @@ -6,6 +6,9 @@ import responses from nose import tools as nt from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser, Permission +from django.core.exceptions import PermissionDenied +from django.http import Http404 from django.core.files.uploadedfile import InMemoryUploadedFile from scripts.update_taxonomies import update_taxonomies @@ -369,6 +372,48 @@ def test_cannot_get_if_preprints_present(self, req, view, preprint, provider_wit assert redirect.url == '/preprint_providers/{}/cannot_delete/'.format(provider_with_preprint.id) assert redirect.status_code == 302 +@pytest.mark.urls('admin.base.urls') +class TestCannotDeleteProviderPermission: + """CannotDeleteProvider must be restricted to users with 'osf.delete_preprintprovider'.""" + + @pytest.fixture() + def provider(self): + return PreprintProviderFactory() + + def test_denied_for_general_user(self, req, provider): + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id) + + def test_denied_for_anonymous(self, provider): + req = RequestFactory().get('/fake_path') + req.user = AnonymousUser() + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id) + + def test_allowed_for_user_with_permission(self, req, user, provider): + permission = Permission.objects.get(codename='delete_preprintprovider') + user.user_permissions.add(permission) + user.save() + + res = views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id) + assert res.status_code == 200 + + def test_allowed_for_superuser(self, req, user, provider): + user.is_superuser = True + user.save() + + res = views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id) + assert res.status_code == 200 + + def test_404_for_missing_provider(self, req, user): + permission = Permission.objects.get(codename='delete_preprintprovider') + user.user_permissions.add(permission) + user.save() + + with pytest.raises(Http404): + views.CannotDeleteProvider.as_view()(req, preprint_provider_id=999999) + + class TestProcessCustomTaxonomy(ProcessCustomTaxonomyMixinBase): @pytest.fixture(autouse=True) diff --git a/admin_tests/quota_recalc/test_views.py b/admin_tests/quota_recalc/test_views.py index cbaa876e015..59b885a8432 100644 --- a/admin_tests/quota_recalc/test_views.py +++ b/admin_tests/quota_recalc/test_views.py @@ -1,20 +1,29 @@ # -*- coding: utf-8 -*- from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser +from django.core.exceptions import PermissionDenied import json import mock from nose import tools as nt from admin.quota_recalc import views from api.base import settings as api_settings -from osf.models import UserQuota +from osf.models import OSFUser, UserQuota from osf_tests.factories import AuthUserFactory, InstitutionFactory, RegionFactory from tests.base import AdminTestCase class TestQuotaRecalcView(AdminTestCase): - @staticmethod - def get_request(view, **kwargs): - return view(RequestFactory().get('/fake_path'), **kwargs) + def setUp(self): + super(TestQuotaRecalcView, self).setUp() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + + def get_request(self, view, **kwargs): + request = RequestFactory().get('/fake_path') + request.user = self.superuser + return view(request, **kwargs) @mock.patch('admin.quota_recalc.views.used_quota') def test_user_create_userquota_record(self, mock_usedquota): @@ -85,7 +94,88 @@ def test_users_create_userquota_record(self, mock_usedquota): res_json3 = json.loads(response3.content) nt.assert_equal(response3.status_code, 200) nt.assert_equal(res_json3['status'], 'OK') - nt.assert_true('2' in res_json3['message']) + nt.assert_true('3' in res_json3['message']) + + +class TestQuotaRecalcPermission(AdminTestCase): + """all_users/user must be restricted to Integrated Admin (is_superuser).""" + + def setUp(self): + super(TestQuotaRecalcPermission, self).setUp() + self.general_user = AuthUserFactory() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + self.target_user = AuthUserFactory() + + def test_all_users_denied_for_general_user(self): + request = RequestFactory().get('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.all_users(request) + + def test_all_users_denied_for_anonymous(self): + request = RequestFactory().get('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.all_users(request) + + @mock.patch('admin.quota_recalc.views.used_quota') + def test_all_users_allowed_for_superuser(self, mock_usedquota): + mock_usedquota.return_value = 0 + request = RequestFactory().get('/fake_path') + request.user = self.superuser + response = views.all_users(request) + nt.assert_equal(response.status_code, 200) + + def test_user_denied_for_general_user(self): + request = RequestFactory().get('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.user(request, guid=self.target_user._id) + + def test_user_denied_for_anonymous(self): + request = RequestFactory().get('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.user(request, guid=self.target_user._id) + + @mock.patch('admin.quota_recalc.views.used_quota') + def test_user_allowed_for_superuser(self, mock_usedquota): + mock_usedquota.return_value = 0 + request = RequestFactory().get('/fake_path') + request.user = self.superuser + response = views.user(request, guid=self.target_user._id) + nt.assert_equal(response.status_code, 200) + + def _invalid_superuser(self, **fields): + # queryset.update() skips OSFUser.save(), which would otherwise recompute is_active + OSFUser.objects.filter(pk=self.superuser.pk).update(**fields) + self.superuser.refresh_from_db() + nt.assert_true(self.superuser.is_superuser) + return self.superuser + + def _assert_denied(self, view, user, **kwargs): + request = RequestFactory().get('/fake_path') + request.user = user + with nt.assert_raises(PermissionDenied): + view(request, **kwargs) + + def test_all_users_denied_for_inactive_superuser(self): + user = self._invalid_superuser(is_active=False) + self._assert_denied(views.all_users, user) + + def test_all_users_denied_for_unregistered_superuser(self): + user = self._invalid_superuser(is_registered=False) + self._assert_denied(views.all_users, user) + + def test_user_denied_for_inactive_superuser(self): + user = self._invalid_superuser(is_active=False) + self._assert_denied(views.user, user, guid=self.target_user._id) + + def test_user_denied_for_unregistered_superuser(self): + user = self._invalid_superuser(is_registered=False) + self._assert_denied(views.user, user, guid=self.target_user._id) class TestCalculateQuota(AdminTestCase): diff --git a/admin_tests/rdm_statistics/test_views.py b/admin_tests/rdm_statistics/test_views.py index 670cfcc31d7..5f60160a58f 100644 --- a/admin_tests/rdm_statistics/test_views.py +++ b/admin_tests/rdm_statistics/test_views.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- from nose import tools as nt from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser +from django.core.exceptions import PermissionDenied from tests.base import AdminTestCase from osf_tests.factories import ( AuthUserFactory, + BookmarkCollectionFactory, InstitutionFactory, ProjectFactory ) @@ -14,6 +17,7 @@ from osf.models.user import Institution from admin.rdm_statistics import views +from framework.auth import cron_signed_url from mock import patch import datetime @@ -179,7 +183,6 @@ def test_get_context_data(self, **kwargs): nt.assert_true('current_date' in ctx) nt.assert_true('user' in ctx) nt.assert_true('provider_data_array' in ctx) - nt.assert_true('token' in ctx) class TestImageView(AdminTestCase): """test ImageView""" @@ -350,9 +353,6 @@ def tearDown(self): self.user.delete() self.institution1.delete() -def test_simple_auth(): - access_key_hexa = '2a85563b2b0f7d3168199f475365f57da1d56e4bb2ce2b7044eb058ae5e287637e7c636a772682d92c8d6b1830b9a97c5a5dc3de7016c60bde4baa7cc3b38aeb' - nt.assert_true(views.simple_auth(access_key_hexa)) def test_get_start_date(): end_date = datetime.datetime.now() @@ -408,7 +408,8 @@ def setUp(self): self.request = RequestFactory().get('/fake_path') self.view = views.GatherView() self.view = setup_user_view(self.view, self.request, user=self.user) - self.view.kwargs = {'institution_id': self.institution1.id, 'access_token': '2A85563B2B0F7D3168199F475365F57DA1D56E4BB2CE2B7044EB058AE5E287637E7C636A772682D92C8D6B1830B9A97C5A5DC3DE7016C60BDE4BAA7CC3B38AEB'.lower()} + ts, signature = cron_signed_url.generate_signed_params() + self.view.kwargs = {'institution_id': self.institution1.id, 'ts': ts, 'signature': signature} def tearDown(self): super(TestGatherView, self).tearDown() @@ -426,8 +427,14 @@ def test_get(self, *args, **kwargs): # metadata addon is now enabled by default, so we have 3 providers nt.assert_equal(len(resp), 3) + def test_get_forbidden_with_invalid_signature(self): + # GatherView.get() reads ts/signature from self.kwargs, set in setUp() + self.view.kwargs['signature'] = 'deadbeef' + resp = json.loads(self.view.get(self.request).content) + nt.assert_equal(resp['state'], 'fail') + def test_send_stat_mail(self, *args, **kwargs): - nt.assert_equal(views.send_stat_mail(self.request).status_code, 200) + nt.assert_equal(views.send_stat_mail_core(self.request).status_code, 200) def test_send_error_mail(self, *args, **kwargs): ret = views.send_error_mail(Exception()) @@ -505,3 +512,99 @@ def test_get_graphs(self, mock_sessionget): provider='osfstorage' ) nt.assert_equal(result['content-type'], 'image/png') + + +class TestSendStatMailPermission(AdminTestCase): + """send_stat_mail (No.86, /statistics/test/mail/) must be restricted to Integrated Admin (is_superuser).""" + + def setUp(self): + super(TestSendStatMailPermission, self).setUp() + self.general_user = AuthUserFactory() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + + def test_send_stat_mail_denied_for_general_user(self): + request = RequestFactory().get('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.send_stat_mail(request) + + def test_send_stat_mail_denied_for_anonymous(self): + request = RequestFactory().get('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.send_stat_mail(request) + + def test_send_stat_mail_allowed_for_superuser(self): + request = RequestFactory().get('/fake_path') + request.user = self.superuser + response = views.send_stat_mail(request) + nt.assert_equal(response.status_code, 200) + + def test_send_stat_mail_denied_for_inactive_superuser(self): + # queryset.update() skips OSFUser.save(), which would otherwise recompute is_active + OSFUser.objects.filter(pk=self.superuser.pk).update(is_active=False) + self.superuser.refresh_from_db() + nt.assert_true(self.superuser.is_superuser) + request = RequestFactory().get('/fake_path') + request.user = self.superuser + with nt.assert_raises(PermissionDenied): + views.send_stat_mail(request) + + def test_send_stat_mail_denied_for_unregistered_superuser(self): + OSFUser.objects.filter(pk=self.superuser.pk).update(is_registered=False) + self.superuser.refresh_from_db() + nt.assert_true(self.superuser.is_superuser) + request = RequestFactory().get('/fake_path') + request.user = self.superuser + with nt.assert_raises(PermissionDenied): + views.send_stat_mail(request) + + +class TestIndexViewPermission(AdminTestCase): + """IndexView must be restricted to Integrated Admin or Institutional Admin.""" + + def setUp(self): + super(TestIndexViewPermission, self).setUp() + self.superuser = AuthUserFactory() + self.superuser.is_superuser = True + self.superuser.save() + self.institution_admin = AuthUserFactory() + self.institution_admin.is_staff = True + self.institution_admin.save() + self.general_user = AuthUserFactory() + + def test_denied_for_general_user(self): + request = RequestFactory().get('/fake_path') + request.user = self.general_user + with nt.assert_raises(PermissionDenied): + views.IndexView.as_view()(request) + + def test_denied_for_anonymous(self): + request = RequestFactory().get('/fake_path') + request.user = AnonymousUser() + with nt.assert_raises(PermissionDenied): + views.IndexView.as_view()(request) + + def test_allowed_for_super_admin(self): + request = RequestFactory().get('/fake_path') + request.user = self.superuser + view = views.IndexView() + view.request = request + nt.assert_true(view.test_func()) + + def test_allowed_for_institution_admin(self): + request = RequestFactory().get('/fake_path') + request.user = self.institution_admin + view = views.IndexView() + view.request = request + nt.assert_true(view.test_func()) + + def test_find_bookmark_collection(self): + """Collection has no 'is_deleted' field; filtering on it raised FieldError.""" + # conftest mocks out new_bookmark_collection for speed, so the bookmark + # collection normally created with the user has to be added here. + bookmark_collection = BookmarkCollectionFactory(creator=self.superuser) + view = views.IndexView() + nt.assert_equal(view.find_bookmark_collection(self.superuser), bookmark_collection) diff --git a/admin_tests/registration_providers/test_views.py b/admin_tests/registration_providers/test_views.py index 9025c46b68c..e2bc66a9f1d 100644 --- a/admin_tests/registration_providers/test_views.py +++ b/admin_tests/registration_providers/test_views.py @@ -2,6 +2,9 @@ import pytest from django.test import RequestFactory +from django.contrib.auth.models import AnonymousUser, Permission +from django.core.exceptions import PermissionDenied +from django.http import Http404 from osf_tests.factories import ( AuthUserFactory, @@ -207,3 +210,79 @@ def test_post(self, view, req, schema, provider): res = view.post(req) assert res.status_code == 302 assert provider.schemas.get(id=schema_id) + + +@pytest.mark.urls('admin.base.urls') +class TestChangeSchemaPermission: + """ChangeSchema must be restricted to users with 'osf.change_registrationprovider'.""" + + @pytest.fixture() + def provider(self): + return RegistrationProviderFactory() + + def test_get_denied_for_general_user(self, req, provider): + with pytest.raises(PermissionDenied): + views.ChangeSchema.as_view()(req, registration_provider_id=provider.id) + + def test_get_denied_for_anonymous(self, provider): + req = RequestFactory().get('/fake_path') + req.user = AnonymousUser() + with pytest.raises(PermissionDenied): + views.ChangeSchema.as_view()(req, registration_provider_id=provider.id) + + def test_get_allowed_for_user_with_permission(self, req, user, provider): + permission = Permission.objects.get(codename='change_registrationprovider') + user.user_permissions.add(permission) + user.save() + + res = views.ChangeSchema.as_view()(req, registration_provider_id=provider.id) + assert res.status_code == 200 + + def test_get_allowed_for_superuser(self, req, user, provider): + user.is_superuser = True + user.save() + + res = views.ChangeSchema.as_view()(req, registration_provider_id=provider.id) + assert res.status_code == 200 + + +@pytest.mark.urls('admin.base.urls') +class TestCannotDeleteProviderPermission: + """CannotDeleteProvider must be restricted to users with 'osf.delete_registrationprovider'.""" + + @pytest.fixture() + def provider(self): + return RegistrationProviderFactory() + + def test_denied_for_general_user(self, req, provider): + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id) + + def test_denied_for_anonymous(self, provider): + req = RequestFactory().get('/fake_path') + req.user = AnonymousUser() + with pytest.raises(PermissionDenied): + views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id) + + def test_allowed_for_user_with_permission(self, req, user, provider): + permission = Permission.objects.get(codename='delete_registrationprovider') + user.user_permissions.add(permission) + user.save() + + res = views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id) + assert res.status_code == 200 + + def test_allowed_for_superuser(self, req, user, provider): + user.is_superuser = True + user.save() + + res = views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id) + assert res.status_code == 200 + + def test_404_for_missing_provider(self, req, user): + permission = Permission.objects.get(codename='delete_registrationprovider') + user.user_permissions.add(permission) + user.save() + + with pytest.raises(Http404): + views.CannotDeleteProvider.as_view()(req, registration_provider_id=999999) diff --git a/api_tests/nodes/views/test_node_addons.py b/api_tests/nodes/views/test_node_addons.py index 459731af87c..660da131d84 100644 --- a/api_tests/nodes/views/test_node_addons.py +++ b/api_tests/nodes/views/test_node_addons.py @@ -1015,8 +1015,8 @@ def _settings_kwargs(self, node, user_settings): def _mock_folder_result(self): return { 'name': 'a.bucket', - 'path': 'a.bucket', - 'id': 'a.bucket' + 'path': '/', + 'id': 'a.bucket:/' } @mock.patch('addons.s3.models.get_bucket_names') diff --git a/framework/auth/cron_signed_url.py b/framework/auth/cron_signed_url.py new file mode 100644 index 00000000000..868e82cf526 --- /dev/null +++ b/framework/auth/cron_signed_url.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +"""HMAC-signed, time-limited URL auth for cron-triggered admin endpoints. + +Used by No.1 (admin/rdm_custom_storage_location/views.py:external_acc_update) +and No.82 (admin/rdm_statistics/views.py:GatherView), which are called by +cron.sh scripts with no interactive user/session. A signed URL looks like +`...///`, where `ts` is a microsecond-resolution Unix +timestamp and `signature` is HMAC(secret, ts). + +Security comes from two independent checks, both required: + 1. authenticity - signature must match HMAC(CRON_SIGNED_URL_SECRET, ts) + 2. freshness - ts must be within CRON_SIGNED_URL_TTL_SECONDS of "now" + +Timestamp precision (microseconds) does not itself add security - it is a +customer requirement, not a defense mechanism. Both checks above are what +actually make an intercepted URL become useless after CRON_SIGNED_URL_TTL_SECONDS. +""" +import hmac +import time + +from website import settings + + +def _sign(ts): + return hmac.new( + key=settings.CRON_SIGNED_URL_SECRET.encode('utf-8'), + msg=str(ts).encode('utf-8'), + digestmod=settings.CRON_SIGNED_URL_HMAC_ALGORITHM, + ).hexdigest() + + +def generate_signed_params(): + """Returns (ts, signature) as strings, ready to drop into a URL path. + + ts is a microsecond-resolution Unix timestamp (str), per customer + requirement (see module docstring). + """ + ts = str(int(time.time() * 1000000)) + return ts, _sign(ts) + + +def verify_signed_params(ts, signature, ttl_seconds=None): + """Verifies a (ts, signature) pair produced by generate_signed_params(). + + Returns True only if signature matches ts AND ts is within ttl_seconds + of now (in either direction). Returns False for any malformed input + instead of raising. + """ + if ttl_seconds is None: + ttl_seconds = settings.CRON_SIGNED_URL_TTL_SECONDS + if not ts or not signature: + return False + try: + ts_us = int(ts) + except (TypeError, ValueError): + return False + expected_signature = _sign(ts) + if not hmac.compare_digest(expected_signature, signature.lower()): + return False + now_us = int(time.time() * 1000000) + ttl_us = int(ttl_seconds * 1000000) + return abs(now_us - ts_us) <= ttl_us diff --git a/osf/management/commands/generate_signed_cron_token.py b/osf/management/commands/generate_signed_cron_token.py new file mode 100644 index 00000000000..98c09cab48f --- /dev/null +++ b/osf/management/commands/generate_signed_cron_token.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from django.core.management.base import BaseCommand + +from framework.auth.cron_signed_url import generate_signed_params + + +class Command(BaseCommand): + """Prints "/" for building a B(2) signed cron URL. + + Used by admin/rdm_statistics/cron.sh (No.82) and + admin/rdm_custom_storage_location/cron.sh (No.1) to mint a fresh + signed URL segment immediately before each curl call. See + framework.auth.cron_signed_url for the verification side. + """ + help = 'Print "/" for a B(2) signed cron URL.' + + def handle(self, *args, **options): + ts, signature = generate_signed_params() + self.stdout.write('{}/{}'.format(ts, signature)) diff --git a/osf/models/files.py b/osf/models/files.py index a53cc81f0ef..cdb8d42d838 100644 --- a/osf/models/files.py +++ b/osf/models/files.py @@ -557,7 +557,7 @@ def update(self, revision, data, user=None, save=True): return version def serialize(self): - newest_version = self.versions.all().last() + newest_version = self.versions.all().first() if not newest_version: return dict(self._serialize(), **{ @@ -571,6 +571,8 @@ def serialize(self): 'locked': self.locked._id if self.locked else None, }) + oldest_version = self.versions.all().last() + return dict(self._serialize(), **{ 'size': newest_version.size, 'downloads': self.get_download_count(), @@ -579,7 +581,7 @@ def serialize(self): 'version': newest_version.identifier if newest_version else None, 'contentType': newest_version.content_type if newest_version else None, 'modified': newest_version.external_modified.isoformat() if newest_version.external_modified else None, - 'created': self.versions.all().first().external_modified.isoformat() if self.versions.all().first().external_modified else None, + 'created': oldest_version.external_modified.isoformat() if oldest_version and oldest_version.external_modified else None, }) def restore(self, recursive=True, parent=None, save=True, deleted_on=None): diff --git a/osf_tests/management_commands/test_generate_signed_cron_token.py b/osf_tests/management_commands/test_generate_signed_cron_token.py new file mode 100644 index 00000000000..821d616c75c --- /dev/null +++ b/osf_tests/management_commands/test_generate_signed_cron_token.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +import io +import pytest +from django.core.management import call_command + +from framework.auth import cron_signed_url + + +@pytest.mark.django_db +class TestGenerateSignedCronToken: + + def test_prints_ts_slash_signature(self): + out = io.StringIO() + call_command('generate_signed_cron_token', stdout=out) + output = out.getvalue().strip() + assert output.count('/') == 1 + ts, signature = output.split('/') + assert ts.isdigit() + assert len(signature) > 0 + + def test_printed_pair_verifies_successfully(self): + out = io.StringIO() + call_command('generate_signed_cron_token', stdout=out) + ts, signature = out.getvalue().strip().split('/') + assert cron_signed_url.verify_signed_params(ts, signature) diff --git a/osf_tests/test_files.py b/osf_tests/test_files.py index c8adac024ba..e8cdb9caa79 100644 --- a/osf_tests/test_files.py +++ b/osf_tests/test_files.py @@ -1,4 +1,7 @@ +import datetime + import pytest +from django.utils import timezone from django.contrib.contenttypes.models import ContentType @@ -8,7 +11,8 @@ from osf_tests.factories import ( UserFactory, ProjectFactory, - RegionFactory + RegionFactory, + FileVersionFactory, ) pytestmark = pytest.mark.django_db @@ -92,3 +96,51 @@ def test_file_update_respects_region(project, user, create_test_file): ) assert new_region != original_region assert new_version.region == new_region + + +def test_file_serialize_no_versions(project): + from addons.googledrive.models import GoogleDriveFile + + # GoogleDriveFile uses File.serialize directly (no override) + gd_file = GoogleDriveFile(name='empty.txt', target=project) + gd_file.save() + + result = gd_file.serialize() + + # newest_version is None → all version-derived fields are None + assert result['size'] is None + assert result['version'] is None + assert result['modified'] is None + assert result['created'] is None + assert result['contentType'] is None + + +def test_file_serialize_newest_version(project): + from addons.googledrive.models import GoogleDriveFile + + # GoogleDriveFile uses File.serialize directly (no override) + gd_file = GoogleDriveFile(name='multi_version.txt', target=project) + gd_file.save() + + # Create v1 first (older DB timestamp), smaller size + older_time = timezone.now() - datetime.timedelta(days=1) + v1 = FileVersionFactory(identifier='1', size=1000) + v1.external_modified = older_time + v1.save() + gd_file.add_version(v1) + + # Create v2 second (newer DB timestamp), larger size + newer_time = timezone.now() + v2 = FileVersionFactory(identifier='2', size=5000) + v2.external_modified = newer_time + v2.save() + gd_file.add_version(v2) + + result = gd_file.serialize() + + # newest_version = versions.all().first() ordered by -created → v2 + assert result['size'] == 5000 + assert result['version'] == '2' + assert result['modified'] == newer_time.isoformat() + # created comes from oldest_version (versions.all().last()) → v1 + assert result['created'] == older_time.isoformat() diff --git a/requirements.txt b/requirements.txt index c9ee19ae23b..f3574223512 100644 --- a/requirements.txt +++ b/requirements.txt @@ -133,3 +133,5 @@ Babel==2.5.1 Flask-Babel==1.0.0 rsa==3.1.4 flask-compress==1.15 + +dataclasses==0.8 \ No newline at end of file diff --git a/scripts/migration/migrate_fix_fileinfo_quota.py b/scripts/migration/migrate_fix_fileinfo_quota.py new file mode 100644 index 00000000000..73464b870d5 --- /dev/null +++ b/scripts/migration/migrate_fix_fileinfo_quota.py @@ -0,0 +1,461 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +migrate_fix_fileinfo_quota.py + +Fix missing or zero-size FileInfo records and recalculate UserQuota. + +Background +---------- +During Move/Copy operations, if a system error occurs after the file has been +written but before the FileInfo hook completes, the resulting OsfStorageFile may +have no FileInfo row (or a row with file_size=0). This causes UserQuota.used to +be under-counted for the affected project creator. + +This script: +1. Finds all non-deleted OsfStorageFile nodes that are missing a FileInfo record + OR have a FileInfo record with file_size=0. +2. Populates file_size from the latest FileVersion.size (which is authoritative). +3. Collects all unique project creators affected by the above. +4. Calls update_user_used_quota(user) for each affected user to re-sync + UserQuota.used from the actual FileInfo totals. + Storage type is determined per-project (NII_STORAGE or CUSTOM_STORAGE). +5. Exports a CSV report of all processed records. + +Error Handling (fail-fast): + The script fails fast. As soon as it hits a record that cannot be fully fixed — + its size cannot be determined, or its project creator / storage_type cannot be + resolved — or any unexpected error, it logs THAT single record and raises + immediately, so an administrator can fix that record and re-run. It does NOT + scan the whole dataset to collect and print a list of every problem. This + behaviour is identical in dry-run and in a real run. + + All DB updates run inside a single transaction (transaction.atomic). If an error + is raised part-way through a real run, every change is rolled back and the + database is left untouched (no partially-applied fixes). + + Note on file size: size is read from the latest FileVersion. A genuine + file_size == 0 is a valid size and a FileInfo record with file_size=0 is + created/updated; only an indeterminable size (None) is treated as an error. + +Usage: + # Fix and export results to CSV + python -m scripts.migration.migrate_fix_fileinfo_quota + python -m scripts.migration.migrate_fix_fileinfo_quota --output result.csv + + # Preview changes and export to CSV without writing to DB (dry run) + python -m scripts.migration.migrate_fix_fileinfo_quota --dry + python -m scripts.migration.migrate_fix_fileinfo_quota --dry --output preview.csv +""" + +import os +import csv +import logging +import argparse +from datetime import datetime + +import django + +# Setup Django before importing models +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'api.base.settings') +django.setup() + +from django.db import transaction +from django.db.models import Q + +from addons.osfstorage.models import OsfStorageFile +from osf.models import FileInfo, OSFUser +from website.util.quota import update_user_used_quota, get_project_storage_type +from scripts import utils as script_utils +from django.contrib.contenttypes.models import ContentType + +logger = logging.getLogger(__name__) + +RESULT_FIELDNAMES = [ + # osf_file_id : _id (hex string) of the OsfStorageFile / BaseFileNode record + # fileinfo_id : integer PK of the FileInfo record inserted/updated (empty in dry_run) + # fileinfo__id : _id (hex string) of the FileInfo record inserted/updated (empty in dry_run) + # fileinfo_file_id: integer PK of BaseFileNode — the value stored as file_id (FK) in the FileInfo table + # fileinfo_created: True = new FileInfo row INSERT-ed, False = existing row UPDATE-d (empty in dry_run) + # action : would_update (dry-run) | created | updated + # quota_recalculated: True if quota was recalculated for this file's creator + 'osf_file_id', 'file_name', + 'fileinfo_id', 'fileinfo__id', 'fileinfo_file_id', + 'file_size_bytes', 'project_guid', 'creator_id', 'storage_type', + 'fileinfo_created', 'action', 'quota_recalculated', 'dry_run', +] + + +# --- Scanning Operations --- + +def get_broken_file_nodes(): + """Return queryset of OsfStorageFile nodes that: + - are not deleted (deleted_on is None) + - either have no FileInfo, or have FileInfo with file_size == 0 + + Optimized using a LEFT OUTER JOIN query to avoid subquery bottlenecks with large datasets. + """ + return ( + OsfStorageFile.objects + .filter(deleted_on=None, deleted_by_id=None) + .filter(Q(fileinfo__isnull=True) | Q(fileinfo__file_size=0)) + # NOTE: Do NOT use prefetch_related('target') here. + # BaseFileNode uses TypedModel + GenericForeignKey; Django's GFK prefetch cache + # is incompatible with TypedModel proxy resolution and returns None even when + # target_content_type_id is valid. Use direct lookup instead (see resolve_target). + ) + + +def _bulk_get_latest_version_sizes(osf_file_ids): + """Fetch the latest FileVersion size for each OsfStorageFile in bulk. + Returns a dict mapping osf_file.id -> size (int or None). + """ + if not osf_file_ids: + return {} + + # Access the M2M through-model (BaseFileVersionsThrough) without importing it directly. + # Fields: basefilenode_id (FK → BaseFileNode), fileversion_id (FK → FileVersion) + ThroughModel = OsfStorageFile.versions.through + + # Single JOIN query: fetches all through-entries + related FileVersion data at once. + entries = ( + ThroughModel.objects + .filter(basefilenode_id__in=osf_file_ids) + .select_related('fileversion') + ) + + # Group by file and pick the version with the highest numeric identifier. + # result maps: file_node_id → (version_number_int, size) + best = {} # file_node_id -> (ver_num, size) + for entry in entries: + fv = entry.fileversion + file_node_id = entry.basefilenode_id + + # identifier is a text field ('1', '2', ...) — parse to int for correct ordering + try: + ver_num = int(fv.identifier) + except (TypeError, ValueError): + ver_num = -1 # non-numeric identifiers rank last + + if file_node_id not in best or ver_num > best[file_node_id][0]: + # Resolve size: prefer fv.size, fall back to metadata dict + size = fv.size + if size is None and isinstance(fv.metadata, dict): + try: + size = int(fv.metadata.get('size')) + except (TypeError, ValueError): + size = None + best[file_node_id] = (ver_num, size) + + return {file_id: info[1] for file_id, info in best.items()} + + +def scan_broken_files(): + """Scan for OsfStorageFile nodes with missing or zero-size FileInfo. + + Fails fast: the moment a record is found that cannot be fully fixed (size cannot + be determined, or its project creator / storage_type cannot be resolved), this + raises so the administrator can fix that one record and re-run. No DB writes. + Returns a list of fully-fixable row dicts when every record is OK. + """ + broken_files = list(get_broken_file_nodes()) + total = len(broken_files) + logger.info('Scanning OsfStorageFile records...') + logger.info('Found {} file node(s) with missing or zero-size FileInfo.'.format(total)) + + # Bulk-fetch latest version sizes to avoid N+1 queries + file_ids = [f.id for f in broken_files] + size_map = _bulk_get_latest_version_sizes(file_ids) + + rows = [] + for osf_file in broken_files: + size = size_map.get(osf_file.id) + + # Fail fast: the size must be determinable. (A genuine file_size == 0 is a + # valid size and is NOT an error — only an indeterminable size is.) + if size is None: + raise RuntimeError( + 'File {} (_id={}) — could not determine size from any FileVersion. ' + 'Fix this record and re-run.'.format(osf_file.name, osf_file._id) + ) + + # Resolve the project target, then its creator / guid / storage_type. + project_guid = '' + creator_id = '' + storage_type_value = None + try: + # Direct lookup via ContentType to avoid GFK+TypedModel prefetch incompatibility. + # osf_file.target (GFK accessor) returns None when used after prefetch_related + # because TypedModel proxy resolution breaks the GFK prefetch cache. + target = None + if osf_file.target_content_type_id and osf_file.target_object_id: + ct = ContentType.objects.get_for_id(osf_file.target_content_type_id) + model_class = ct.model_class() + if model_class: + target = model_class.objects.filter(id=osf_file.target_object_id).first() + + if target is not None: + first_guid = target.guids.values_list('_id', flat=True).first() + project_guid = first_guid or '' + creator = getattr(target, 'creator', None) + if creator: + creator_id = creator._id + storage_type_value = get_project_storage_type(target) + except Exception as e: + # Fail fast: do not swallow unexpected errors. + logger.error('Could not resolve target/creator for file {}: {} — aborting (fail fast).'.format( + osf_file._id, e + )) + raise + + # Fail fast: creator and storage_type must be resolvable to recalculate quota. + if not creator_id: + raise RuntimeError( + 'File {} (_id={}, project_guid={}) — creator could not be resolved ' + '(target missing or has no creator). Fix this record and re-run.'.format( + osf_file.name, osf_file._id, project_guid + ) + ) + if storage_type_value is None: + raise RuntimeError( + 'File {} (_id={}, project_guid={}) — project storage_type could not be ' + 'determined. Fix this record and re-run.'.format( + osf_file.name, osf_file._id, project_guid + ) + ) + + rows.append({ + 'osf_file_id': osf_file._id, # hex _id string of OsfStorageFile + 'fileinfo_file_id': osf_file.id, # integer PK stored as file_id FK in FileInfo table + 'file_name': osf_file.name, + 'file_size_bytes': size, + 'project_guid': project_guid, + 'creator_id': creator_id, + 'storage_type': storage_type_value, + 'osf_file': osf_file, + }) + + logger.info('Scan complete. {} record(s) ready to fix.'.format(len(rows))) + return rows + + +def print_results(rows): + """Display scanned records in a formatted summary table.""" + if not rows: + print(' No broken FileInfo records found.') + return + + def _disp(value): + # Display helper: never let None (or any value) break str formatting. + return '' if value is None else str(value) + + print('\n' + '-' * 160) + print(' {:<26} {:<16} {:<22} {:<18} {:<16} {:<14} {:<18}'.format( + 'OSF_FILE_ID (_id)', 'FILEINFO_FILE_ID', 'FILE_NAME', 'PROJECT_GUID', 'CREATOR_ID', 'STORAGE_TYPE', 'SIZE (bytes)' + )) + print('-' * 160) + for r in rows: + print(' {:<26} {:<16} {:<22} {:<18} {:<16} {:<14} {:<18}'.format( + _disp(r.get('osf_file_id')), + _disp(r.get('fileinfo_file_id')), + _disp(r.get('file_name'))[:20], + _disp(r.get('project_guid')), + _disp(r.get('creator_id')), + _disp(r.get('storage_type')), + _disp(r.get('file_size_bytes')), + )) + print('-' * 160) + print(' Total: {} record(s) would be updated.\n'.format(len(rows))) + + +# --- Updating Operations --- + +def apply_fixes(rows, dry_run=False): + """Apply FileInfo fixes and recalculate UserQuota for affected users. + + This function wraps all database modifications in a single database transaction + when dry_run is False. If any unhandled exception occurs, the transaction is + rolled back, ensuring database consistency. + + If dry_run is True, no database writes are performed. + Returns the list of result rows with 'action', 'quota_recalculated', and 'dry_run' fields added. + """ + if dry_run: + logger.info('[DRY RUN] Simulating FileInfo updates (no DB writes)...') + return _apply_fixes_internal(rows, dry_run=True) + else: + logger.info('Applying FileInfo updates (wrapped in database transaction)...') + with transaction.atomic(): + return _apply_fixes_internal(rows, dry_run=False) + + +def _apply_fixes_internal(rows, dry_run=False): + """Internal helper to apply fixes, called inside/outside transaction. + + Every row here is already fully fixable (scan_broken_files fails fast otherwise), + so each row gets a FileInfo update and contributes to a UserQuota recalculation. + """ + result_rows = [] + affected_user_storage = set() # set of (creator_id, storage_type) + fixed = 0 + + for row in rows: + # scan stores the osf_file object on the row; fall back to a lookup just in case. + osf_file = row.get('osf_file') + if osf_file is None: + osf_file = OsfStorageFile.objects.filter(_id=row['osf_file_id']).first() + if osf_file is None: + # Fail fast: a scanned row must resolve to a file. + raise RuntimeError('File {} not found when applying fix.'.format(row['osf_file_id'])) + + fileinfo_id = '' + fileinfo__id = '' + fileinfo_created = '' + if not dry_run: + fileinfo_obj, created = FileInfo.objects.update_or_create( + file=osf_file, + defaults={'file_size': row['file_size_bytes']}, + ) + fileinfo_id = fileinfo_obj.id # integer PK of FileInfo + fileinfo__id = fileinfo_obj._id # hex _id string of FileInfo + fileinfo_created = created # True = INSERT, False = UPDATE + + creator_id = row['creator_id'] + storage_type = row['storage_type'] + affected_user_storage.add((creator_id, storage_type)) + + action = 'would_update' if dry_run else ('created' if fileinfo_created else 'updated') + + logger.info('{} {} (osf_file_id={}, fileinfo_id={}, fileinfo__id={}, fileinfo_file_id={}, created={}) file_size={} bytes storage_type={}'.format( + '[DRY RUN]' if dry_run else 'Fixed:', + row['file_name'], row['osf_file_id'], + fileinfo_id, fileinfo__id, row['fileinfo_file_id'], + fileinfo_created, row['file_size_bytes'], storage_type, + )) + + result_rows.append({ + 'osf_file_id': row['osf_file_id'], + 'fileinfo_file_id': row['fileinfo_file_id'], + 'file_name': row['file_name'], + 'file_size_bytes': row['file_size_bytes'], + 'project_guid': row['project_guid'], + 'creator_id': row['creator_id'], + 'storage_type': row['storage_type'], + 'fileinfo_id': fileinfo_id, + 'fileinfo__id': fileinfo__id, + 'fileinfo_created': fileinfo_created, + 'action': action, + 'quota_recalculated': False, # will be updated below after quota recalc + 'dry_run': dry_run, + }) + fixed += 1 + + logger.info('FileInfo backfill complete. fixed={} (dry={})'.format(fixed, dry_run)) + + # --- Recalculate UserQuota.used for all affected (user, storage_type) pairs --- + logger.info('Recalculating UserQuota.used for {} unique (user, storage_type) pair(s).'.format( + len(affected_user_storage) + )) + + quota_updated = 0 + recalculated_user_storage = set() + + for user_id, storage_type in sorted(affected_user_storage): + try: + user = OSFUser.load(user_id) + if user is None: + # Fail fast: a creator_id resolved during scan must load here. + raise RuntimeError( + 'User {} (storage_type={}) could not be loaded for quota recalc.'.format( + user_id, storage_type + ) + ) + + if not dry_run: + update_user_used_quota(user, storage_type=storage_type, is_recalculating_quota=True) + + logger.info('{} Recalculated quota for user {} (storage_type={})'.format( + '[DRY RUN]' if dry_run else '', user_id, storage_type + )) + quota_updated += 1 + recalculated_user_storage.add((user_id, storage_type)) + except Exception as e: + # Fail fast: re-raise so transaction.atomic() rolls back ALL FileInfo updates. + logger.error('Failed to recalculate quota for user {} (storage_type={}): {} — ' + 'rolling back transaction (fail fast).'.format(user_id, storage_type, e)) + raise + + logger.info('Quota recalculation complete. updated={} (dry={})'.format(quota_updated, dry_run)) + + # Update quota_recalculated flag in result rows + for row in result_rows: + if dry_run: + row['quota_recalculated'] = 'would_recalculate' + elif (row['creator_id'], row['storage_type']) in recalculated_user_storage: + row['quota_recalculated'] = True + + return result_rows + + +# --- CSV Export --- + +def save_result_csv(result_rows, output_path): + """Save full results to CSV file.""" + with open(output_path, 'w', newline='', encoding='utf-8-sig') as f: + writer = csv.DictWriter(f, fieldnames=RESULT_FIELDNAMES) + writer.writeheader() + writer.writerows(result_rows) + total = len(result_rows) + print('\n Exported {} result(s) to: {}'.format(total, output_path)) + + +# --- Main --- + +def main(): + ts = datetime.now().strftime('%Y%m%d_%H%M%S') + default_output = f'migrate_fix_fileinfo_quota_{ts}.csv' + + parser = argparse.ArgumentParser( + description='Fix missing/zero-size FileInfo records and recalculate UserQuota.', + ) + parser.add_argument( + '--output', '-o', + default=default_output, + help=f'Output CSV file path (default: {default_output})', + ) + parser.add_argument( + '--dry', action='store_true', default=False, + help='Run in dry-run mode (do not commit updates to the database).', + ) + + args = parser.parse_args() + + if not args.dry: + script_utils.add_file_logger(logger, __file__) + + # Configure console logging format + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s', + ) + + # 1. Scan for broken records + rows = scan_broken_files() + + if not rows: + print('No broken FileInfo records found.') + return + + # 2. Print summary table + print_results(rows) + + # 3. Apply fixes (or simulate in dry-run mode). scan_broken_files() has already + # failed fast on any unfixable record, so every row here is fully fixable. + result_rows = apply_fixes(rows, dry_run=args.dry) + + # 4. Export results to CSV + save_result_csv(result_rows, args.output) + + +if __name__ == '__main__': + main() diff --git a/tests/test_addons.py b/tests/test_addons.py index e6f18031b56..2254a03fa75 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -25,7 +25,7 @@ DraftRegistrationFactory, ExportDataLocationFactory, ) -from website import settings +from website import settings, mails from api.base import settings as api_settings from addons.base import views from addons.github.exceptions import ApiError @@ -113,12 +113,33 @@ def test_auth_download(self): data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data'] assert_equal(data['auth'], views.make_auth(self.user)) assert_equal(data['credentials'], self.node_addon.serialize_waterbutler_credentials()) - assert_equal(data['settings'], self.node_addon.serialize_waterbutler_settings()) + expected_settings = self.node_addon.serialize_waterbutler_settings().copy() + expected_settings['max_file_size'] = 100 + assert_equal(data['settings'], expected_settings) expected_url = furl.furl(self.node.api_url_for('create_waterbutler_log', _absolute=True, _internal=True)) observed_url = furl.furl(data['callback_url']) observed_url.port = expected_url.port assert_equal(expected_url, observed_url) + def test_auth_max_file_size_honors_high_upload_limit_tag(self): + """A user with the `high_upload_limit` system tag gets the high max_file_size + sent to WaterButler.""" + self.user.add_system_tag('high_upload_limit') + self.user.save() + url = self.build_url() + with mock.patch.object(self.node_addon.config, 'high_max_file_size', 500, create=True): + res = self.app.get(url, auth=self.user.auth) + data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data'] + assert_equal(data['settings']['max_file_size'], 500) + + def test_auth_max_file_size_without_high_upload_limit_tag(self): + """An untagged user keeps the addon's ordinary limit even when a high limit exists.""" + url = self.build_url() + with mock.patch.object(self.node_addon.config, 'high_max_file_size', 500, create=True): + res = self.app.get(url, auth=self.user.auth) + data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data'] + assert_equal(data['settings']['max_file_size'], 100) + def test_auth_render_action_returns_200(self): url = self.build_url(action='render') res = self.app.get(url, auth=self.user.auth) @@ -153,7 +174,9 @@ def test_auth_bad_cookie(self): data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data'] assert_equal(data['auth'], views.make_auth(self.user)) assert_equal(data['credentials'], self.node_addon.serialize_waterbutler_credentials()) - assert_equal(data['settings'], self.node_addon.serialize_waterbutler_settings()) + expected_settings = self.node_addon.serialize_waterbutler_settings().copy() + expected_settings['max_file_size'] = 100 + assert_equal(data['settings'], expected_settings) expected_url = furl.furl(self.node.api_url_for('create_waterbutler_log', _absolute=True, _internal=True)) observed_url = furl.furl(data['callback_url']) observed_url.port = expected_url.port @@ -1019,6 +1042,99 @@ def test_add_folder_osfstorage_log(self, mock_basefilenode): assert_equal(self.node.logs.count(), nlogs + 1) assert('urls' not in self.node.logs.filter(action='osf_storage_file_added')[0].params) + @mock.patch('website.mails.send_mail') + def test_create_waterbutler_log_quota_exceeded_error(self, mock_send_mail): + url = self.node.api_url_for('create_waterbutler_log') + payload = self.build_payload( + action='move', + metadata={'path': 'pizza'}, + source={ + 'provider': 'github', + 'materialized': '/src.txt', + 'path': '/src.txt', + 'name': 'src.txt', + 'nid': self.node._id, + }, + destination={ + 'provider': 'github', + 'materialized': '/dest.txt', + 'path': '/dest.txt', + 'name': 'dest.txt', + 'nid': self.node._id, + }, + errors=[''] + ) + res = self.app.put_json(url, payload, headers={'Content-Type': 'application/json'}) + assert_equal(res.status_code, 200) + assert_true(mock_send_mail.called) + call_args = mock_send_mail.call_args[1] + assert_equal(call_args['error_info'], {'type': 'quota_exceeded'}) + assert_equal(mock_send_mail.call_args[0][1], mails.FILE_OPERATION_FAILED) + + @mock.patch('website.mails.send_mail') + def test_create_waterbutler_log_oversized_error(self, mock_send_mail): + url = self.node.api_url_for('create_waterbutler_log') + payload = self.build_payload( + action='move', + metadata={'path': 'pizza'}, + source={ + 'provider': 'github', + 'materialized': '/src.txt', + 'path': '/src.txt', + 'name': 'src.txt', + 'nid': self.node._id, + }, + destination={ + 'provider': 'github', + 'materialized': '/dest.txt', + 'path': '/dest.txt', + 'name': 'dest.txt', + 'nid': self.node._id, + }, + errors=[''] + ) + res = self.app.put_json(url, payload, headers={'Content-Type': 'application/json'}) + assert_equal(res.status_code, 200) + assert_true(mock_send_mail.called) + call_args = mock_send_mail.call_args[1] + assert_equal(call_args['error_info'], { + 'type': 'oversized', + 'oversized_files': [{'name': 'large.zip', 'size': 2000000000}], + 'max_size': 1000000000 + }) + assert_equal(mock_send_mail.call_args[0][1], mails.FILE_OPERATION_FAILED) + + @mock.patch('addons.base.views.logger.warning') + @mock.patch('website.mails.send_mail') + def test_create_waterbutler_log_unparseable_error(self, mock_send_mail, mock_logger_warning): + url = self.node.api_url_for('create_waterbutler_log') + payload = self.build_payload( + action='move', + metadata={'path': 'pizza'}, + source={ + 'provider': 'github', + 'materialized': '/src.txt', + 'path': '/src.txt', + 'name': 'src.txt', + 'nid': self.node._id, + }, + destination={ + 'provider': 'github', + 'materialized': '/dest.txt', + 'path': '/dest.txt', + 'name': 'dest.txt', + 'nid': self.node._id, + }, + errors=[''] + ) + res = self.app.put_json(url, payload, headers={'Content-Type': 'application/json'}) + assert_equal(res.status_code, 200) + assert_true(mock_send_mail.called) + assert_true(mock_logger_warning.called) + call_args = mock_send_mail.call_args[1] + assert_equal(call_args['error_info'], None) + assert_equal(mock_send_mail.call_args[0][1], mails.FILE_OPERATION_FAILED) + class TestAddonLogsDifferentProvider(OsfTestCase): def setUp(self): diff --git a/tests/test_cron_signed_url.py b/tests/test_cron_signed_url.py new file mode 100644 index 00000000000..d77169a13ce --- /dev/null +++ b/tests/test_cron_signed_url.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +import time +import mock +from nose.tools import * # noqa (PEP8 asserts) + +from framework.auth import cron_signed_url + + +class TestCronSignedUrl: + + def setup_method(self, method): + self.secret_patch = mock.patch('website.settings.CRON_SIGNED_URL_SECRET', 'test-secret') + self.ttl_patch = mock.patch('website.settings.CRON_SIGNED_URL_TTL_SECONDS', 100) + self.secret_patch.start() + self.ttl_patch.start() + + def teardown_method(self, method): + self.secret_patch.stop() + self.ttl_patch.stop() + + def test_generate_then_verify_succeeds(self): + ts, signature = cron_signed_url.generate_signed_params() + assert_true(cron_signed_url.verify_signed_params(ts, signature)) + + def test_ts_is_microsecond_precision(self): + before_us = int(time.time() * 1000000) + ts, _ = cron_signed_url.generate_signed_params() + after_us = int(time.time() * 1000000) + assert_true(before_us <= int(ts) <= after_us) + + def test_tampered_signature_is_rejected(self): + ts, signature = cron_signed_url.generate_signed_params() + flipped_char = '0' if signature[0] != '0' else '1' + tampered = flipped_char + signature[1:] + assert_false(cron_signed_url.verify_signed_params(ts, tampered)) + + def test_tampered_timestamp_is_rejected(self): + ts, signature = cron_signed_url.generate_signed_params() + tampered_ts = str(int(ts) + 1) + assert_false(cron_signed_url.verify_signed_params(tampered_ts, signature)) + + def test_expired_timestamp_is_rejected(self): + with mock.patch('time.time', return_value=time.time() - 200): + ts, signature = cron_signed_url.generate_signed_params() + assert_false(cron_signed_url.verify_signed_params(ts, signature)) + + def test_future_timestamp_beyond_ttl_is_rejected(self): + with mock.patch('time.time', return_value=time.time() + 200): + ts, signature = cron_signed_url.generate_signed_params() + assert_false(cron_signed_url.verify_signed_params(ts, signature)) + + def test_non_numeric_timestamp_is_rejected(self): + assert_false(cron_signed_url.verify_signed_params('not-a-number', 'deadbeef')) + + def test_missing_signature_is_rejected(self): + ts, _ = cron_signed_url.generate_signed_params() + assert_false(cron_signed_url.verify_signed_params(ts, '')) + + def test_wrong_secret_is_rejected(self): + ts, signature = cron_signed_url.generate_signed_params() + with mock.patch('website.settings.CRON_SIGNED_URL_SECRET', 'a-different-secret'): + assert_false(cron_signed_url.verify_signed_params(ts, signature)) diff --git a/tests/test_oauth.py b/tests/test_oauth.py index 74ce6195df1..af4002d5bf7 100644 --- a/tests/test_oauth.py +++ b/tests/test_oauth.py @@ -38,6 +38,7 @@ ) from django.test import RequestFactory from admin.rdm_custom_storage_location import views as customstoragelocation_update +from framework.auth import cron_signed_url SILENT_LOGGERS = ['oauthlib', 'requests_oauthlib'] @@ -1050,11 +1051,12 @@ def test_callback_with_institution(self, mock_drive_refresh): assert_equal(account.oauth_key, 'mock_access_token') assert_equal(account.provider_id, 'mock_provider_id') req = RequestFactory().get('http://localhost:8001/customstoragelocation/external_acc_update/') - res = customstoragelocation_update.external_acc_update(req,access_token='d610ef95f0b0f5868f13919b8ed64070b9acb9c19b8da9f2c514ed938203ec3e236c9cad4f4146bdf22b4e79cf0d92f6d4f4c996d236c6b0ee79a1336b26afb7') - assert_equal(res.status_code,200) + ts, signature = cron_signed_url.generate_signed_params() + res = customstoragelocation_update.external_acc_update(req, ts=ts, signature=signature) + assert_equal(res.status_code, 200) assert_equal(res.content.decode(), 'Done') - res = customstoragelocation_update.external_acc_update(req,access_token='b610ef95f0b0f5868f13919b8ed64070b9acb9c19b8da9f2c514ed938203ec3e236c9cad4f4146bdf22b4e79cf0d92f6d4f4c996d236c6b0ee79a1336b26afb7') - assert_equal(res.status_code,200) + res = customstoragelocation_update.external_acc_update(req, ts=ts, signature='deadbeef') + assert_equal(res.status_code, 200) assert_not_equal(res.content.decode(), 'Done') @responses.activate diff --git a/tests/test_quota.py b/tests/test_quota.py index 2fc183213d0..d3e29977ba8 100644 --- a/tests/test_quota.py +++ b/tests/test_quota.py @@ -268,7 +268,7 @@ def setUp(self): ) self.file.save() - def test_add_file_info(self): + def test_file_info_skipped_for_osfstorage_signal(self): file_info_query = FileInfo.objects.filter(file=self.file) assert_false(file_info_query.exists()) @@ -294,9 +294,8 @@ def test_add_file_info(self): ) file_info_list = FileInfo.objects.filter(file=self.file).all() - assert_equal(file_info_list.count(), 1) - file_info = file_info_list.first() - assert_equal(file_info.file_size, 1000) + # FileInfo was not created + assert_equal(file_info_list.count(), 0) def test_update_file_info(self): file_info = FileInfo(file=self.file, file_size=1000) @@ -386,7 +385,7 @@ def setUp(self): target_content_type_id=2 ) - def test_add_first_file(self): + def test_first_file_userquota_not_created_for_osfstorage_signal(self): assert_false(UserQuota.objects.filter(user=self.project_creator).exists()) quota.update_used_quota( @@ -414,9 +413,7 @@ def test_add_first_file(self): storage_type=UserQuota.NII_STORAGE, user=self.project_creator ).all() - assert_equal(len(user_quota), 1) - user_quota = user_quota[0] - assert_equal(user_quota.used, 1000) + assert_equal(len(user_quota), 0) def test_add_first_file_custom_storage(self): assert_false(UserQuota.objects.filter(user=self.project_creator).exists()) @@ -453,11 +450,9 @@ def test_add_first_file_custom_storage(self): storage_type=UserQuota.CUSTOM_STORAGE, user=self.project_creator ).all() - assert_equal(len(user_quota), 1) - user_quota = user_quota[0] - assert_equal(user_quota.used, 1200) + assert_equal(len(user_quota), 0) - def test_add_file(self): + def test_userquota_unchanged_for_osfstorage_signal(self): UserQuota.objects.create( user=self.project_creator, storage_type=UserQuota.NII_STORAGE, @@ -492,7 +487,7 @@ def test_add_file(self): ).all() assert_equal(len(user_quota), 1) user_quota = user_quota[0] - assert_equal(user_quota.used, 6500) + assert_equal(user_quota.used, 5500) def test_add_file_custom_storage(self): UserQuota.objects.create( @@ -536,7 +531,7 @@ def test_add_file_custom_storage(self): ).all() assert_equal(len(user_quota), 1) user_quota = user_quota[0] - assert_equal(user_quota.used, 6700) + assert_equal(user_quota.used, 5500) def test_add_file_negative_size(self): quota.update_used_quota( @@ -545,9 +540,9 @@ def test_add_file_negative_size(self): user=self.user, event_type=FileLog.FILE_ADDED, payload={ - 'provider': 'osfstorage', + 'provider': 's3compatinstitutions', 'metadata': { - 'provider': 'osfstorage', + 'provider': 's3compatinstitutions', 'name': 'testfile', 'materialized': '/filename', 'path': self.file._id, @@ -599,6 +594,51 @@ def test_delete_file(self): ) assert_equal(user_quota.used, 4500) + @mock.patch('website.util.quota.check_select_for_update') + def test_delete_file_without_select_for_update(self, mock_check_select_for_update): + # Forces node_removed() into the `else` branch (FileInfo.objects.get + # without select_for_update), instead of relying on whether the test + # happens to run inside an atomic transaction. + mock_check_select_for_update.return_value = False + + UserQuota.objects.create( + user=self.project_creator, + storage_type=UserQuota.NII_STORAGE, + max_quota=api_settings.DEFAULT_MAX_QUOTA, + used=5500 + ) + FileInfo.objects.create(file=self.file, file_size=1000) + + self.file.deleted_on = datetime.datetime.now() + self.file.deleted_by = self.user + self.file.type = 'osf.trashedfile' + self.file.save() + + quota.update_used_quota( + self=None, + target=self.node, + user=self.user, + event_type=FileLog.FILE_REMOVED, + payload={ + 'provider': 'osfstorage', + 'metadata': { + 'provider': 'osfstorage', + 'name': 'testfile', + 'materialized': '/filename', + 'path': self.file._id, + 'kind': 'file', + 'extra': {} + } + } + ) + + mock_check_select_for_update.assert_called() + user_quota = UserQuota.objects.get( + storage_type=UserQuota.NII_STORAGE, + user=self.project_creator + ) + assert_equal(user_quota.used, 4500) + def test_delete_file_custom_storage(self): UserQuota.objects.create( user=self.project_creator, @@ -987,9 +1027,9 @@ def test_edit_file_negative_size(self): user=self.user, event_type=FileLog.FILE_UPDATED, payload={ - 'provider': 'osfstorage', + 'provider': 's3compatinstitutions', 'metadata': { - 'provider': 'osfstorage', + 'provider': 's3compatinstitutions', 'name': 'testfile', 'materialized': '/filename', 'path': self.file._id, @@ -1358,7 +1398,7 @@ def test_delete_file_with_Amazon_S3_Compatible_Storage_for_Institution(self): if check_select_for_update(): mock_file_info.objects.filter.return_value.select_for_update.return_value.get.return_value = FileInfo( file=self.base_file_node, file_size=1000) - mock_user_quota.objects.filter.return_value.select_for_update.return_value.first.return_value = UserQuota( + mock_user_quota.objects.filter.return_value.select_for_update.return_value.get.return_value = UserQuota( user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE, max_quota=api_settings.DEFAULT_MAX_QUOTA, @@ -1366,12 +1406,14 @@ def test_delete_file_with_Amazon_S3_Compatible_Storage_for_Institution(self): ) else: mock_file_info.objects.get.return_value = FileInfo(file=self.base_file_node, file_size=1000) - mock_user_quota.objects.filter.return_value.first.return_value = UserQuota( + _real_user_quota = UserQuota( user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE, max_quota=api_settings.DEFAULT_MAX_QUOTA, used=5500 ) + mock_user_quota.objects.get.return_value = _real_user_quota + mock_user_quota.objects.filter.return_value.first.return_value = _real_user_quota with mock.patch('website.util.quota.BaseFileNode', mock_base_file_node): with mock.patch('website.util.quota.FileInfo', mock_file_info): with mock.patch('website.util.quota.UserQuota', mock_user_quota): @@ -1414,7 +1456,7 @@ def test_delete_folder_with_Amazon_S3_Compatible_Storage_for_Institution(self): if check_select_for_update(): mock_file_info.objects.filter.return_value.select_for_update.return_value.get.return_value = FileInfo( file=self.base_file_node, file_size=1500) - mock_user_quota.objects.filter.return_value.select_for_update.return_value.first.return_value = UserQuota( + mock_user_quota.objects.filter.return_value.select_for_update.return_value.get.return_value = UserQuota( user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE, max_quota=api_settings.DEFAULT_MAX_QUOTA, @@ -1422,12 +1464,14 @@ def test_delete_folder_with_Amazon_S3_Compatible_Storage_for_Institution(self): ) else: mock_file_info.objects.get.return_value = FileInfo(file=self.base_file_node, file_size=1500) - mock_user_quota.objects.filter.return_value.first.return_value = UserQuota( + _real_user_quota = UserQuota( user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE, max_quota=api_settings.DEFAULT_MAX_QUOTA, used=5500 ) + mock_user_quota.objects.get.return_value = _real_user_quota + mock_user_quota.objects.filter.return_value.first.return_value = _real_user_quota with mock.patch('website.util.quota.BaseFileNode', mock_base_file_node): with mock.patch('website.util.quota.FileInfo', mock_file_info): with mock.patch('website.util.quota.UserQuota', mock_user_quota): @@ -1551,6 +1595,20 @@ def test_default_values(self): assert_equal(response.json['max'], api_settings.DEFAULT_MAX_QUOTA * api_settings.SIZE_UNIT_GB) assert_equal(response.json['used'], 0) + def test_user_guid_and_storage_type_present_and_consistent_with_creator(self): + response = self.app.get( + '{}?payload={payload}&signature={signature}'.format( + self.node.api_url_for('waterbutler_creator_quota'), + **signing.sign_data(signing.default_signer, {}) + ) + ) + assert_equal(response.status_code, 200) + assert_in('user_guid', response.json) + assert_in('storage_type', response.json) + from website.util import quota as quota_util + assert_equal(response.json['user_guid'], self.node.creator._id) + assert_equal(response.json['storage_type'], quota_util.get_project_storage_type(self.node)) + def test_used_half_custom_quota(self): UserQuota.objects.create( storage_type=UserQuota.NII_STORAGE, @@ -1617,6 +1675,15 @@ def test_default_values(self): assert_equal(response.json['max'], api_settings.DEFAULT_MAX_QUOTA * api_settings.SIZE_UNIT_GB) assert_equal(response.json['used'], 0) + def test_user_guid_and_storage_type_present_and_consistent_with_creator(self): + response = self.app.get( + self.node.api_url_for('get_creator_quota'), + auth=self.user.auth + ) + assert_equal(response.status_code, 200) + assert_in('user_guid', response.json) + assert_in('storage_type', response.json) + def test_used_half_custom_quota(self): UserQuota.objects.create( storage_type=UserQuota.NII_STORAGE, diff --git a/tests/test_quota_move_copy.py b/tests/test_quota_move_copy.py new file mode 100644 index 00000000000..49081f7ad72 --- /dev/null +++ b/tests/test_quota_move_copy.py @@ -0,0 +1,287 @@ +import pytest +from unittest import mock +from website.util import quota +from django.db import IntegrityError + +@pytest.fixture +def mock_user(): + user = mock.Mock() + user.id = 1 + user._id = 'abc123' + user.userquota_set.get.side_effect = quota.UserQuota.DoesNotExist + return user + +@pytest.fixture +def mock_node(): + node = mock.Mock(spec=quota.AbstractNode) + node.id = 42 + node.creator = mock.Mock() + node.type = 'osf.node' + return node + +def test_get_file_size_file(): + children = {'children': [{'kind': 'file', 'size': 100}, {'kind': 'file', 'size': 200}]} + assert quota.get_file_size(children) == 300 + +def test_get_file_size_nested(): + children = { + 'children': [ + {'kind': 'file', 'size': 100}, + {'kind': 'folder', 'children': [{'kind': 'file', 'size': 50}]} + ] + } + assert quota.get_file_size(children) == 150 + +@mock.patch('website.util.quota.logger') +def test_get_file_size_none_size_treated_as_zero_and_warns(mock_logger): + """A child with size=None must not crash get_file_size.""" + children = {'children': [{'kind': 'file', 'size': None, 'name': 'unknown.gdoc'}]} + assert quota.get_file_size(children) == 0 + assert mock_logger.warning.called + +@mock.patch('website.util.quota.logger') +def test_get_file_size_negative_size_treated_as_zero_and_warns(mock_logger): + """A negative child size must be treated as 0.""" + children = {'children': [{'kind': 'file', 'size': -1, 'name': 'bad.txt'}]} + assert quota.get_file_size(children) == 0 + assert mock_logger.warning.called + +def test_get_file_size_string_size(): + """Providers that report size as a str (e.g. s3compat HEAD Content-Length header) + must not raise TypeError when compared against 0.""" + children = {'children': [{'kind': 'file', 'size': '100'}, {'kind': 'file', 'size': '200'}]} + assert quota.get_file_size(children) == 300 + +@mock.patch('website.util.quota.logger') +def test_get_file_size_negative_string_size_treated_as_zero_and_warns(mock_logger): + """A negative child size reported as a str must also be treated as 0.""" + children = {'children': [{'kind': 'file', 'size': '-1', 'name': 'bad.txt'}]} + assert quota.get_file_size(children) == 0 + assert mock_logger.warning.called + +@mock.patch('website.util.quota.logger') +def test_get_file_size_mixed_children_none_and_valid(mock_logger): + """A None-size sibling must not swallow the total contributed by valid siblings.""" + children = { + 'children': [ + {'kind': 'file', 'size': 100, 'name': 'ok.txt'}, + {'kind': 'file', 'size': None, 'name': 'unknown.gdoc'}, + ] + } + assert quota.get_file_size(children) == 100 + assert mock_logger.warning.called + +@mock.patch('website.util.quota.ProjectStorageType.objects.get') +@mock.patch('website.util.quota.AbstractNode.objects.get') +@mock.patch('website.util.quota.update_quota') +@pytest.mark.django_db +def test_handle_move_copy_source_extend_dest_osfstorage(mock_update_quota, mock_node_get, mock_pst_get, mock_node): + """source=s3compatinstitutions, dest=osfstorage""" + mock_pst_get.return_value = mock.Mock(storage_type=1) + mock_node_get.return_value = mock_node + + payload = { + 'destination': {'provider': 'osfstorage', 'size': 123, 'children': None}, + 'source': {'nid': 'nid1', 'provider': 's3compatinstitutions'} + } + quota._handle_move_copy(quota.FileLog.FILE_MOVED, mock_node, mock.Mock(), payload) + mock_update_quota.assert_not_called() + +@mock.patch('website.util.quota.ProjectStorageType.objects.get') +@mock.patch('website.util.quota.AbstractNode.objects.get') +@mock.patch('website.util.quota.update_quota') +@pytest.mark.django_db +def test_handle_move_copy_source_osfstorage(mock_update_quota, mock_node_get, mock_pst_get, mock_user): + mock_node = mock.create_autospec(quota.AbstractNode, instance=True) + mock_node.type = 'osf.node' + mock_pst_get.return_value = mock.Mock(storage_type=1) + mock_node_get.return_value = mock_node + + payload = { + 'destination': {'provider': 's3compatinstitutions', 'size': 123}, + 'source': {'provider': 'osfstorage', 'size': 123, 'nid': 'nid1'} + } + quota._handle_move_copy(quota.FileLog.FILE_MOVED, mock_node, mock_user, payload) + mock_update_quota.assert_called_once() + +@mock.patch('website.util.quota.update_quota') +@pytest.mark.django_db +def test_handle_move_copy_no_osfstorage(mock_update_quota, mock_node, mock_user): + payload = { + 'destination': {'provider': 's3compatinstitutions'}, + 'source': {'provider': 's3compatinstitutions'} + } + quota._handle_move_copy(quota.FileLog.FILE_MOVED, mock_node, mock_user, payload) + mock_update_quota.assert_not_called() + +@mock.patch('website.util.quota.ProjectStorageType.objects.get') +@mock.patch('website.util.quota.AbstractNode.objects.get') +@mock.patch('website.util.quota.update_quota') +@pytest.mark.django_db +def test_file_moved_file(mock_update_quota, mock_node_get, mock_pst_get, mock_node): + mock_pst_get.return_value = mock.Mock(storage_type=1) + mock_node_get.return_value = mock_node + payload = { + 'destination': {'size': 123, 'children': None}, + 'source': {'nid': 'nid1', 'provider': 'osfstorage'} + } + quota.file_moved(mock_node, payload) + mock_update_quota.assert_called_once() + +@mock.patch('website.util.quota.ProjectStorageType.objects.get') +@mock.patch('website.util.quota.AbstractNode.objects.get') +@mock.patch('website.util.quota.update_quota') +@pytest.mark.django_db +def test_file_moved_folder(mock_update_quota, mock_node_get, mock_pst_get, mock_node): + mock_pst_get.return_value = mock.Mock(storage_type=1) + mock_node_get.return_value = mock_node + payload = { + 'destination': {'children': [{'kind': 'file', 'size': 100}]}, + 'source': {'nid': 'nid1', 'provider': 'osfstorage'} + } + quota.file_moved(mock_node, payload) + mock_update_quota.assert_called_once() + +@mock.patch('website.util.quota.ProjectStorageType.objects.get', side_effect=Exception) +@mock.patch('website.util.quota.AbstractNode.objects.get') +@mock.patch('website.util.quota.update_quota') +@pytest.mark.django_db +def test_file_moved_exception(mock_update_quota, mock_node_get, mock_pst_get, mock_node): + mock_node_get.return_value = mock_node + payload = { + 'destination': {'size': 123, 'children': None}, + 'source': {'nid': 'nid1', 'provider': 'osfstorage'} + } + with pytest.raises(Exception): + quota.file_moved(mock_node, payload) + mock_update_quota.assert_not_called() + +@mock.patch('website.util.quota.UserQuota.objects.get') +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_add(mock_filter, mock_get, mock_node): + uq = mock.Mock() + uq.used = 0 + mock_get.return_value = uq + mock_filter.return_value.select_for_update.return_value.get.return_value = uq + quota.update_quota(mock_node, 100, 1, add=True) + assert uq.used == 100 + +@mock.patch('website.util.quota.UserQuota.objects.get') +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_subtract(mock_filter, mock_get, mock_node): + uq = mock.Mock() + uq.used = 200 + mock_get.return_value = uq + mock_filter.return_value.select_for_update.return_value.get.return_value = uq + quota.update_quota(mock_node, 50, 1, add=False) + assert uq.used == 150 + +@mock.patch('website.util.quota.check_select_for_update', return_value=False) +@mock.patch('website.util.quota.transaction.atomic') +@mock.patch('website.util.quota.UserQuota.objects.create') +@mock.patch('website.util.quota.UserQuota.objects.filter') +@mock.patch('website.util.quota.UserQuota.objects.get', side_effect=quota.UserQuota.DoesNotExist) +@pytest.mark.django_db +def test_update_quota_userquota_not_exist(mock_get, mock_filter, mock_create, mock_atomic, mock_node): + mock_filter.return_value.select_for_update.return_value.get.side_effect = quota.UserQuota.DoesNotExist + quota.update_quota(mock_node, 50, 1, add=True) + mock_get.assert_called_once() + +@mock.patch('website.util.quota.UserQuota.objects.get', side_effect=Exception) +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_other_exception(mock_filter, mock_get, mock_node): + mock_filter.return_value.select_for_update.return_value.get.side_effect = Exception + with pytest.raises(Exception): + quota.update_quota(mock_node, 50, 1, add=True) + mock_get.assert_not_called() + +@mock.patch('website.util.quota._handle_move_copy') +def test_update_used_quota_move_event(mock_handle_move_copy, mock_node, mock_user): + payload = { + 'destination': {'provider': 's3compatinstitutions'}, + 'source': {'provider': 'osfstorage'} + } + quota.update_used_quota(None, mock_node, mock_user, quota.FileLog.FILE_MOVED, payload) + mock_handle_move_copy.assert_called_once() + +@mock.patch('website.util.quota._handle_move_copy') +def test_update_used_quota_dest_osfstorage_returns_early(mock_handle_move_copy, mock_node, mock_user): + payload = { + 'destination': {'provider': 'osfstorage'} + } + result = quota.update_used_quota(None, mock_node, mock_user, quota.FileLog.FILE_MOVED, payload) + assert result is None + mock_handle_move_copy.assert_not_called() + +@mock.patch('website.util.quota.check_select_for_update', return_value=True) +@mock.patch('website.util.quota.transaction.atomic') +@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError) +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_integrityerror_select_for_update_add( + mock_filter, mock_create, mock_atomic, mock_check, mock_node +): + uq = mock.Mock() + uq.used = 10 + mock_filter.return_value.select_for_update.return_value.get.side_effect = [ + quota.UserQuota.DoesNotExist, + uq + ] + quota.update_quota(mock_node, 5, 1, add=True) + assert mock_filter.return_value.select_for_update.return_value.get.call_count == 2 + assert uq.used == 15 + uq.save.assert_called_once() + +@mock.patch('website.util.quota.check_select_for_update', return_value=True) +@mock.patch('website.util.quota.transaction.atomic') +@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError) +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_integrityerror_select_for_update_subtract( + mock_filter, mock_create, mock_atomic, mock_check, mock_node +): + uq = mock.Mock() + uq.used = 10 + mock_filter.return_value.select_for_update.return_value.get.side_effect = [ + quota.UserQuota.DoesNotExist, + uq + ] + quota.update_quota(mock_node, 3, 1, add=False) + assert mock_filter.return_value.select_for_update.return_value.get.call_count == 1 + assert uq.used == 10 + uq.save.assert_not_called() + +@mock.patch('website.util.quota.check_select_for_update', return_value=False) +@mock.patch('website.util.quota.transaction.atomic') +@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError) +@mock.patch('website.util.quota.UserQuota.objects.get') +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_integrityerror_get_add( + mock_filter, mock_get, mock_create, mock_atomic, mock_check, mock_node +): + uq = mock.Mock() + uq.used = 20 + mock_get.side_effect = [quota.UserQuota.DoesNotExist, uq] + quota.update_quota(mock_node, 8, 1, add=True) + assert uq.used == 28 + uq.save.assert_called_once() + +@mock.patch('website.util.quota.check_select_for_update', return_value=False) +@mock.patch('website.util.quota.transaction.atomic') +@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError) +@mock.patch('website.util.quota.UserQuota.objects.get') +@mock.patch('website.util.quota.UserQuota.objects.filter') +@pytest.mark.django_db +def test_update_quota_integrityerror_get_subtract( + mock_filter, mock_get, mock_create, mock_atomic, mock_check, mock_node +): + uq = mock.Mock() + uq.used = 30 + mock_get.side_effect = [quota.UserQuota.DoesNotExist, uq] + quota.update_quota(mock_node, 12, 1, add=False) + assert uq.used == 30 + uq.save.assert_not_called() diff --git a/website/project/views/quota.py b/website/project/views/quota.py index 88814f268a0..8f78fbd5d83 100644 --- a/website/project/views/quota.py +++ b/website/project/views/quota.py @@ -17,10 +17,11 @@ def get_quota_from_pid(pid): """Auxiliary function for getting the quota from a project ID. Used on requests by waterbutler and the user (from browser).""" node = AbstractNode.load(pid) - max_quota, used_quota = quota.get_quota_info( - node.creator, quota.get_project_storage_type(node) - ) + storage_type = quota.get_project_storage_type(node) + max_quota, used_quota = quota.get_quota_info(node.creator, storage_type) return { 'max': max_quota * api_settings.SIZE_UNIT_GB, - 'used': used_quota + 'used': used_quota, + 'user_guid': node.creator._id, + 'storage_type': storage_type, } diff --git a/website/settings/__init__.py b/website/settings/__init__.py index 38da961d50b..e533b3a4619 100644 --- a/website/settings/__init__.py +++ b/website/settings/__init__.py @@ -21,7 +21,7 @@ if not DEV_MODE: from . import local from . import defaults - for setting in ('WATERBUTLER_JWE_SECRET', 'WATERBUTLER_JWE_SALT', 'WATERBUTLER_JWT_SECRET', 'JWT_SECRET', 'DEFAULT_HMAC_SECRET', 'POPULAR_LINKS_NODE', 'NEW_AND_NOTEWORTHY_LINKS_NODE', 'SENSITIVE_DATA_SALT', 'SENSITIVE_DATA_SECRET'): + for setting in ('WATERBUTLER_JWE_SECRET', 'WATERBUTLER_JWE_SALT', 'WATERBUTLER_JWT_SECRET', 'JWT_SECRET', 'DEFAULT_HMAC_SECRET', 'CRON_SIGNED_URL_SECRET', 'POPULAR_LINKS_NODE', 'NEW_AND_NOTEWORTHY_LINKS_NODE', 'SENSITIVE_DATA_SALT', 'SENSITIVE_DATA_SECRET'): assert getattr(local, setting, None) and getattr(local, setting, None) != getattr(defaults, setting, None), '{} must be specified in local.py when DEV_MODE is False'.format(setting) diff --git a/website/settings/defaults.py b/website/settings/defaults.py index e808456a88a..12f9a27782d 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -330,6 +330,13 @@ def parent_dir(path): DEFAULT_HMAC_SECRET = 'changeme' DEFAULT_HMAC_ALGORITHM = hashlib.sha256 + +# signed-URL auth for cron-triggered admin endpoints (No.1 external_acc_update, +# No.82 GatherView). Deliberately separate from DEFAULT_HMAC_SECRET (WaterButler<->OSF) +# so a leak of one does not affect the other. +CRON_SIGNED_URL_SECRET = 'changeme' +CRON_SIGNED_URL_HMAC_ALGORITHM = hashlib.sha256 +CRON_SIGNED_URL_TTL_SECONDS = 100 WATERBUTLER_URL = 'http://localhost:7777' WATERBUTLER_INTERNAL_URL = WATERBUTLER_URL diff --git a/website/static/img/institutions/banners/edugain-logo.png b/website/static/img/institutions/banners/edugain-logo.png new file mode 100644 index 00000000000..812f470d210 Binary files /dev/null and b/website/static/img/institutions/banners/edugain-logo.png differ diff --git a/website/static/img/institutions/shields-rounded-corners/edugain-shield-rounded-corners.png b/website/static/img/institutions/shields-rounded-corners/edugain-shield-rounded-corners.png new file mode 100644 index 00000000000..a42be72e902 Binary files /dev/null and b/website/static/img/institutions/shields-rounded-corners/edugain-shield-rounded-corners.png differ diff --git a/website/static/img/institutions/shields/edugain-shield.png b/website/static/img/institutions/shields/edugain-shield.png new file mode 100644 index 00000000000..a42be72e902 Binary files /dev/null and b/website/static/img/institutions/shields/edugain-shield.png differ diff --git a/website/static/js/fangorn.js b/website/static/js/fangorn.js index bb0e87f283f..d6b6b44829c 100644 --- a/website/static/js/fangorn.js +++ b/website/static/js/fangorn.js @@ -89,6 +89,10 @@ var OPERATIONS = { } }; +var MESSAGE_MAP = { + 'quota_exceeded': gettext('You do not have enough available quota.'), +}; + var isInUploadFolderProcess = false; var isOngoingUploadFolder = false; @@ -706,6 +710,111 @@ function checkConflictsRename(tb, item, name, cb) { cb('replace'); } +/** + * Recursively finds all individual files that exceed maxSizeBytes. + * Returns an array of offending item objects. + */ +function _findOversizedFiles(item, maxSizeBytes) { + var oversized = []; + if (item.kind === 'file') { + var size = item.data.size || 0; + if (size > 0 && size > maxSizeBytes) { + oversized.push(item); + } + } else if (item.kind === 'folder' && item.children && item.children.length > 0) { + for (var i = 0; i < item.children.length; i++) { + oversized = oversized.concat(_findOversizedFiles(item.children[i], maxSizeBytes)); + } + } + return oversized; +} + +/** + * Helper function to reject a conflict item (e.g. if file size or quota check fails) + * and advance the sync move/copy queue. + */ +function _rejectConflictItem(tb, from, to, notRenameOp, conflict) { + from.inProgress = false; + if (notRenameOp) { + addFileStatus(tb, from, false, '', '', conflict); + // Use object identity (not data.id): for MOVE, .done() replaces from.data + // in-place so from.data.id changes before .always() runs. + tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function(file) { + return file !== from; + }); + if (!tb.pendingReadyFiles.length) { + doSyncMove(tb, to.data.provider); + } + } +} + +/** + * True when `node` itself, or any folder anywhere beneath it, has not been + * lazy-loaded yet — meaning the subtree's total size cannot be known client-side. + * + * The file browser lazy-loads a single level at a time, so an expanded folder can + * still contain sub-folders whose own children have never been fetched. Summing + * only what happens to be loaded would under-count the freed size. + */ +function hasUnloadedFolder(node) { + if (node.kind === 'folder' && !node.load) { + return true; + } + return (node.children || []).some(hasUnloadedFolder); +} + +/** + * Size freed up at the destination by an overwrite, in bytes. + * + * Mirrors WaterButler's get_replaced_size(): only conflict === 'replace' overwrites + * anything ('keep' renames the incoming item; omitted conflict defaults to 'warn' — + * both free 0 bytes), and only a same-kind destination item is replaced (osfstorage + * allows a file and folder to share a name). + * + * Returns null if the destination folder's subtree isn't fully loaded (size unknown) — + * callers must skip the client-side check and defer to the server. + */ +function getReplacedSize(to, from, conflict) { + if (conflict !== 'replace') { + return 0; + } + var existing = (to.children || []).filter(function(child) { + return child.data.name === from.data.name && child.kind === from.kind; + })[0]; + if (!existing) { + return 0; + } + if (from.kind !== 'folder') { + return parseInt(existing.data.size, 10) || 0; + } + if (hasUnloadedFolder(existing)) { + // Part of the subtree is not lazy-loaded yet; total size unknown, let the + // server-side check decide. + return null; + } + return getAllChildren(existing).reduce(function(total, child) { + return child.kind === 'file' ? total + (parseInt(child.data.size, 10) || 0) : total; + }, 0); +} + +// max_file_size skip narrows by provider only (browser can't know region); +// server gives the authoritative 413 if it turns out not to qualify. +function shouldSkipSizeCheck(operation, fromProvider, destProvider) { + return operation.status === 'move' && fromProvider === destProvider; +} + +function isSameUserQuota(srcQuota, destQuota) { + return !!(srcQuota && destQuota && + srcQuota.user_guid !== undefined && srcQuota.user_guid !== null && + srcQuota.user_guid === destQuota.user_guid && + srcQuota.storage_type === destQuota.storage_type); +} + +function quotaCheckExceeds(destQuota, totalSize, replacedSize) { + replacedSize = replacedSize || 0; + return (destQuota.used + totalSize - replacedSize) > destQuota.max; +} + function doItemOp(operation, to, from, rename, conflict) { var tb = this; // dismiss old modal immediately to prevent button mashing @@ -714,6 +823,7 @@ function doItemOp(operation, to, from, rename, conflict) { var filesRemaining; var inConflictsQueue = false; var syncMoves; + var showQuotaUsageAlert = false; var notRenameOp = typeof rename === 'undefined'; if (notRenameOp) { @@ -749,7 +859,113 @@ function doItemOp(operation, to, from, rename, conflict) { return; } + if (operation !== OPERATIONS.RENAME) { + var destProvider = to.data.provider; + + // Sum size of all files in the item being moved/copied + var totalMoveSize = 0; + if (from.kind === 'file') { + totalMoveSize = parseInt(from.data.size, 10) || 0; + } else { + getAllChildren(from).forEach(function(child) { + if (child.kind === 'file') { + totalMoveSize += parseInt(child.data.size, 10) || 0; + } + }); + } + + var skipSizeCheck = shouldSkipSizeCheck(operation, from.data.provider, destProvider); + + if (!skipSizeCheck) { + var destMaxSize = to.data && to.data.accept && to.data.accept.maxSize; + if (destMaxSize) { + var maxSizeBytes = destMaxSize * 1000000; + var oversizedFiles = _findOversizedFiles(from, maxSizeBytes); + if (oversizedFiles.length > 0) { + var maxSizeDisplay = $osf.humanFileSize(maxSizeBytes, true); + oversizedFiles.forEach(function(oversized) { + var displaySize = $osf.humanFileSize(oversized.data.size, true); + $osf.growl(sprintf( + gettext('File「%1$s」is too large (%2$s). Max file size is %3$s.'), + oversized.data.name, displaySize, maxSizeDisplay + )); + }); + _rejectConflictItem(tb, from, to, notRenameOp, conflict); + return; + } + } + } + + // Quota context is resolved only after the size check above has run, mirroring + // the server's run_pre_checks order: an oversized file is rejected with 413 + // before creator_quota is ever fetched. + var destQuota = null; + var srcQuota = null; + var skipQuotaCheck = false; + if (destProvider === 'osfstorage' && totalMoveSize > 0) { + // Sequential async:false calls (kept in sync with doItemOp's control flow) + var destQuotaResp = $.ajax({ + async: false, + method: 'GET', + url: to.data.nodeApiUrl + 'get_creator_quota/' + }); + // Only a MOVE from osfstorage needs src quota — it serves just to detect a + // same-record move. A falsy srcQuota means "adds usage". + var srcQuotaResp = null; + if (operation.status === 'move' && from.data.provider === 'osfstorage') { + // Same node ⇒ same UserQuota record: reuse the response already in hand + // instead of asking the very same endpoint twice. + srcQuotaResp = (from.data.nodeApiUrl === to.data.nodeApiUrl) ? destQuotaResp : $.ajax({ + async: false, + method: 'GET', + url: from.data.nodeApiUrl + 'get_creator_quota/' + }); + } + if (destQuotaResp.responseJSON && (!srcQuotaResp || srcQuotaResp.responseJSON)) { + destQuota = destQuotaResp.responseJSON; + srcQuota = srcQuotaResp ? srcQuotaResp.responseJSON : null; + skipQuotaCheck = (operation.status === 'move' && + isSameUserQuota(srcQuota, destQuota)); + } + // A failed request leaves destQuota null: the browser evaluates nothing here — + // neither the quota check nor the threshold warning runs below — and defers + // entirely to the server-side check. + } + + // destQuota is only set for an osfstorage destination with something to weigh. + // skipQuotaCheck is the single source of truth for same-UserQuota-record moves: + // by this point that case is already excluded, so nothing below needs to re-check it. + if (destQuota && !skipQuotaCheck) { + var replacedSize = getReplacedSize(to, from, conflict); + var quotaMsgText = ''; + + // replacedSize === null: destination folder not yet loaded, so its size + // is unknown — skip only this blocking check and let the move/copy proceed; + // the server-side check is authoritative. + if (replacedSize !== null && quotaCheckExceeds(destQuota, totalMoveSize, replacedSize)) { + if (from.kind === 'file') { + quotaMsgText = sprintf(gettext('Not enough quota to move/copy the file.')); + } else { + var folderSize = $osf.humanFileSize(totalMoveSize, true); + quotaMsgText = sprintf(gettext('Not enough quota to move/copy. The total size of the folder %1$s.'), folderSize); + } + from.notify.update(quotaMsgText, 'warning', undefined, 3000); + _rejectConflictItem(tb, from, to, notRenameOp, conflict); + return; + } + // Unknown replacedSize is treated as 0 (worst case) so the alert can still fire. + var quotaDelta = totalMoveSize - (replacedSize || 0); + // Shown only after a confirmed success below. The alert announces *crossing* + // the threshold, so an operation that adds no usage must stay silent. + showQuotaUsageAlert = (quotaDelta > 0 && + destQuota.used + quotaDelta > destQuota.max * window.contextVars.threshold); + } + } + var origFrom = Object.assign({}, from); + // Must capture before COPY rebinds `from` to a new clone. + // .done() replaces from.data in-place, so ID-based filtering breaks for MOVE ops. + var originalFrom = from; if (operation === OPERATIONS.COPY) { from = tb.createItem($.extend(true, {status: operation.status}, from.data), to.id); } else { @@ -815,6 +1031,17 @@ function doItemOp(operation, to, from, rename, conflict) { from.data.status = undefined; from.notify.update(sprintf(gettext('Successfully %1$s.') , gettext(operation.passed)), 'success', null, 1000); + if (showQuotaUsageAlert) { + $osf.growl( + gettext('Quota usage alert'), + sprintf( + gettext('You have used more than %1$s%% of your quota.'), + (window.contextVars.threshold * 100) + ), + 'warning' + ); + } + if (xhr.status === 200) { to.children.forEach(function(child) { if (child.data.name === from.data.name && child.id !== from.id) { @@ -854,6 +1081,41 @@ function doItemOp(operation, to, from, rename, conflict) { from.data.status = undefined; } + if (xhr.status === 413 && xhr.responseJSON && xhr.responseJSON.oversized_files) { + var serverMaxSize = xhr.responseJSON.max_size; + var destMaxSize = to.data && to.data.accept && to.data.accept.maxSize; + var maxSizeDisplay = serverMaxSize ? + $osf.humanFileSize(serverMaxSize, true) : + (destMaxSize ? $osf.humanFileSize(destMaxSize * 1000000, true) : null); + xhr.responseJSON.oversized_files.forEach(function(fileObj) { + var displaySize = $osf.humanFileSize(fileObj.size, true); + var msg = maxSizeDisplay ? + sprintf(gettext('File「%1$s」is too large (%2$s). Max file size is %3$s.'), fileObj.name, displaySize, maxSizeDisplay) : + sprintf(gettext('File「%1$s」is too large (%2$s).'), fileObj.name, displaySize); + $osf.growl(msg); + }); + + if (notRenameOp) { + addFileStatus(tb, from, false, '', '', conflict); + } + orderFolder.call(tb, from.parent()); + // NOTE: pendingReadyFiles filter, from.inProgress reset, and doSyncMove + // are all handled by .always() below. Do NOT duplicate them here. + return; + } + if (xhr.status === 406 && xhr.responseJSON && xhr.responseJSON.message) { + var key = xhr.responseJSON.message_key; + var msg = (key && MESSAGE_MAP[key]) ? MESSAGE_MAP[key] : xhr.responseJSON.message; + $osf.growl(msg); + if (notRenameOp) { + addFileStatus(tb, from, false, '', '', conflict); + } + orderFolder.call(tb, from.parent()); + // NOTE: pendingReadyFiles filter, from.inProgress reset, and doSyncMove + // are all handled by .always() below. Do NOT duplicate them here. + return; + } + var message; if (xhr.status !== 500 && xhr.responseJSON && (xhr.responseJSON.message || xhr.responseJSON.message_long)) { @@ -878,7 +1140,9 @@ function doItemOp(operation, to, from, rename, conflict) { orderFolder.call(tb, from.parent()); }).always(function(){ - tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function (file) { return file.data.id !== from.data.id; }); + // Use object identity: .done() replaces from.data in-place (MOVE op), + // making ID-based comparison unreliable by the time .always() runs. + tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function (file) { return file !== originalFrom; }); from.inProgress = false; if (notRenameOp && !tb.pendingReadyFiles.length){ doSyncMove(tb, to.data.provider); @@ -3647,6 +3911,7 @@ function _dropLogic(event, items, folder) { if (toMove.conflicts.length > 0) { tb.syncFileMoveCache[folder.data.provider].conflicts = tb.syncFileMoveCache[folder.data.provider].conflicts || []; + tb.syncFileMoveCache[folder.data.provider].ready = tb.syncFileMoveCache[folder.data.provider].ready || []; toMove.conflicts.forEach(function(item) { tb.syncFileMoveCache[folder.data.provider].conflicts.push({'item' : item, 'folder' : folder}); }); @@ -4412,5 +4677,11 @@ module.exports = { getCopyMode : getCopyMode, showDeleteMultiple : showDeleteMultiple, checkConflicts : checkConflicts, - getPersistentLinkFor: getPersistentLinkFor + getPersistentLinkFor: getPersistentLinkFor, + quotaCheckExceeds: quotaCheckExceeds, + getReplacedSize: getReplacedSize, + isSameUserQuota: isSameUserQuota, + shouldSkipSizeCheck: shouldSkipSizeCheck, + doItemOp: doItemOp, + OPERATIONS: OPERATIONS }; diff --git a/website/static/js/tests/fangorn.test.js b/website/static/js/tests/fangorn.test.js index 10a18c56f1e..c34f3bd5c88 100644 --- a/website/static/js/tests/fangorn.test.js +++ b/website/static/js/tests/fangorn.test.js @@ -14,6 +14,8 @@ var Raven = require('raven-js'); window.contextVars = { osfSupportEmail : 'fake-support@osf.io', + waterbutlerURL : 'https://files.osf.io/', + threshold : 0.9, }; var language = require('js/osfLanguage').projectSettings; @@ -423,6 +425,647 @@ describe('fangorn', () => { assert.equal(Fangorn.showDeleteMultiple([folder, item]), true); }); }); + + describe('shouldSkipSizeCheck', () => { + it('returns true for a move where source and destination provider match', () => { + var operation = {status: 'move'}; + assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 'osfstorage'), true); + }); + + it('returns false for a move where source and destination provider differ', () => { + var operation = {status: 'move'}; + assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 's3'), false); + }); + + it('returns false for a copy even when provider matches', () => { + var operation = {status: 'copy'}; + assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 'osfstorage'), false); + }); + + it('returns false for a copy where provider also differs', () => { + var operation = {status: 'copy'}; + assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 's3'), false); + }); + }); + + describe('isSameUserQuota', () => { + it('returns true when user_guid and storage_type both match', () => { + var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + var dest = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, dest), true); + }); + + it('returns false when user_guid differs', () => { + var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + var dest = {user_guid: 'xyz', storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, dest), false); + }); + + it('returns false when storage_type differs', () => { + var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + var dest = {user_guid: 'abc', storage_type: 'CUSTOM_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, dest), false); + }); + + it('returns false when user_guid is undefined on either side', () => { + var src = {storage_type: 'NII_STORAGE'}; + var dest = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, dest), false); + }); + + it('returns false when user_guid is null on both sides', () => { + var src = {user_guid: null, storage_type: 'NII_STORAGE'}; + var dest = {user_guid: null, storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, dest), false); + }); + + it('returns false when srcQuota is falsy', () => { + var dest = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(null, dest), false); + }); + + it('returns false when destQuota is falsy', () => { + var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'}; + assert.equal(Fangorn.isSameUserQuota(src, null), false); + }); + }); + + describe('quotaCheckExceeds', () => { + it('returns true when used + totalSize exceeds max', () => { + var destQuota = {used: 800, max: 1000}; + assert.equal(Fangorn.quotaCheckExceeds(destQuota, 300, 0), true); + }); + + it('returns false when used + totalSize stays within max', () => { + var destQuota = {used: 800, max: 1000}; + assert.equal(Fangorn.quotaCheckExceeds(destQuota, 100, 0), false); + }); + + it('returns false when used + totalSize lands exactly on max', () => { + var destQuota = {used: 800, max: 1000}; + assert.equal(Fangorn.quotaCheckExceeds(destQuota, 200, 0), false); + }); + + it('subtracts replacedSize before comparing to max', () => { + var destQuota = {used: 800, max: 1000}; + // Without replacedSize this would exceed (800 + 300 > 1000), + // but the 150 freed by the overwrite brings it back within max. + assert.equal(Fangorn.quotaCheckExceeds(destQuota, 300, 150), false); + }); + + it('treats a missing replacedSize as 0', () => { + var destQuota = {used: 800, max: 1000}; + assert.equal(Fangorn.quotaCheckExceeds(destQuota, 300), true); + }); + }); + + describe('getReplacedSize', () => { + // Fixture builder for treebeard-like nodes: getReplacedSize/hasUnloadedFolder + // read top-level `kind`/`load`, unlike the `getItem` helper above which only + // sets `data.kind`. + var makeNode = function(kind, name, opts) { + opts = opts || {}; + return { + kind: kind, + data: { + name: name, + size: opts.size + }, + children: opts.children || [], + load: opts.load !== undefined ? opts.load : true + }; + }; + + it('returns 0 when conflict is not replace', () => { + var to = {children: []}; + var from = makeNode('file', 'a.txt', {size: 10}); + assert.equal(Fangorn.getReplacedSize(to, from, 'keep'), 0); + }); + + it('returns 0 when no existing item matches name and kind', () => { + var to = {children: [makeNode('file', 'other.txt', {size: 10})]}; + var from = makeNode('file', 'a.txt', {size: 10}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 0); + }); + + it('does not match when name is the same but kind differs', () => { + var to = {children: [makeNode('folder', 'a', {})]}; + var from = makeNode('file', 'a', {size: 10}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 0); + }); + + it('returns the existing file size when replacing a file', () => { + var existing = makeNode('file', 'a.txt', {size: 500}); + var to = {children: [existing]}; + var from = makeNode('file', 'a.txt', {size: 10}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 500); + }); + + it('returns 0 when the existing file has no size', () => { + var existing = makeNode('file', 'a.txt', {}); + var to = {children: [existing]}; + var from = makeNode('file', 'a.txt', {size: 10}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 0); + }); + + it('sums the sizes of all files in a fully loaded existing folder', () => { + var child1 = makeNode('file', 'x.txt', {size: 100}); + var child2 = makeNode('file', 'y.txt', {size: 200}); + var subfolder = makeNode('folder', 'sub', {children: [child1, child2], load: true}); + var existing = makeNode('folder', 'a', {children: [subfolder], load: true}); + var to = {children: [existing]}; + var from = makeNode('folder', 'a', {}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 300); + }); + + it('returns null when the existing folder itself has not been lazy-loaded', () => { + var existing = makeNode('folder', 'a', {load: false}); + var to = {children: [existing]}; + var from = makeNode('folder', 'a', {}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), null); + }); + + it('returns null when a nested subfolder has not been lazy-loaded', () => { + var unloadedSub = makeNode('folder', 'sub', {load: false}); + var existing = makeNode('folder', 'a', {children: [unloadedSub], load: true}); + var to = {children: [existing]}; + var from = makeNode('folder', 'a', {}); + assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), null); + }); + }); + }); + + describe('doItemOp', () => { + var ajaxStub; + var growlStub; + var ravenStub; + + beforeEach(() => { + growlStub = sinon.stub($osf, 'growl'); + ravenStub = sinon.stub(Raven, 'captureMessage'); + }); + + afterEach(() => { + if (ajaxStub) { ajaxStub.restore(); } + growlStub.restore(); + ravenStub.restore(); + }); + + // Fixture builder for a fake Treebeard node with just enough of the + // real API (move/parent/notify/add/removeSelf) for doItemOp to run. + function makeParent() { + return { + data: { + nodeUrl: 'https://osf.io/proj1/', + nodeApiUrl: 'https://osf.io/proj1/api/', + nodeId: 'proj1', + permissions: {edit: true}, + provider: 'osfstorage' + }, + sortChildren: sinon.stub() + }; + } + + function makeItem(kind, data, id, parentID) { + return { + kind: kind, + data: $.extend({provider: 'osfstorage'}, data), + children: [], + id: id, + parentID: parentID, + inProgress: false, + notify: {update: sinon.stub()}, + move: sinon.stub(), + add: sinon.stub(), + removeSelf: sinon.stub(), + parent: sinon.stub().returns(makeParent()) + }; + } + + // Fixture builder for a fake Treebeard.controller (`this` inside doItemOp). + function makeTb() { + var tb = { + modal: {dismiss: sinon.stub(), update: sinon.stub()}, + pendingFileOps: [], + pendingReadyFiles: [], + moveStates: [], + uploadStates: [], + syncFileMoveCache: {osfstorage: {conflicts: [], ready: []}, s3compat: {conflicts: [], ready: []}}, + clearMultiselect: sinon.stub(), + redraw: sinon.stub(), + select: sinon.stub().returns({hide: sinon.stub(), css: sinon.stub()}), + options: { + lazyLoadPreprocess: function(resp) { return {data: resp}; } + } + }; + tb.createItem = sinon.stub().callsFake(function(data) { + return makeItem(data.kind, data); + }); + tb.buildTree = sinon.stub().callsFake(function(data) { + return makeItem(data.kind, data); + }); + return tb; + } + + // A minimal fake jQuery Deferred supporting the .done()/.fail()/.always() + // chain doItemOp attaches to the final POST call. + function fakeDeferred(config) { + var obj = { + done: function(cb) { + if (config.status === 'done') { cb.apply(null, config.args || []); } + return obj; + }, + fail: function(cb) { + if (config.status === 'fail') { cb.apply(null, config.args || []); } + return obj; + }, + always: function(cb) { cb(); return obj; } + }; + return obj; + } + + function stubAjax(responder) { + ajaxStub = sinon.stub($, 'ajax').callsFake(responder); + } + + function growlCallWithType(type) { + return growlStub.getCalls().some(function(call) { return call.args[2] === type; }); + } + + it('returns without calling the API when dropped back onto its own parent', () => { + var tb = makeTb(); + var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/'}, 5, undefined); + var from = makeItem('file', {name: 'a.txt', size: 10}, 3, 5); + ajaxStub = sinon.stub($, 'ajax'); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(ajaxStub.called, false); + }); + + it('rejects the move without calling the API when a file exceeds the destination max size', () => { + var tb = makeTb(); + // Different providers: shouldSkipSizeCheck only shortcuts a same-provider move, + // so this keeps the client-side oversized check from being bypassed. + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1); + var from = makeItem('file', {name: 'big.txt', size: 5000000}, 3, 1); + ajaxStub = sinon.stub($, 'ajax'); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(ajaxStub.called, false); + assert.equal(growlStub.called, true); + assert.equal(from.inProgress, false); + }); + + it('rejects the copy without calling the API when it would exceed quota', () => { + var tb = makeTb(); + var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/'}, 5, 1); + var from = makeItem('file', {name: 'a.txt', size: 100}, 3, 1); + stubAjax(function(opts) { + if (opts.method === 'GET') { + return {responseJSON: {used: 950, max: 1000, user_guid: 'u1', storage_type: 'NII_STORAGE'}}; + } + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined); + assert.equal(ajaxStub.callCount, 1); + assert.equal(from.notify.update.called, true); + assert.equal(from.inProgress, false); + }); + + it('skips the quota check and proceeds when source and destination share the same UserQuota record', () => { + var tb = makeTb(); + var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1); + var from = makeItem('file', {nodeApiUrl: 'https://osf.io/from/', name: 'a.txt', size: 999999, path: '/a.txt'}, 3, 1); + var sameQuota = {used: 10, max: 1000000000, user_guid: 'u1', storage_type: 'NII_STORAGE'}; + stubAjax(function(opts) { + if (opts.method === 'GET') { + return {responseJSON: sameQuota}; + } + if (opts.type === 'POST') { + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 'osfstorage', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'}, + 'success', + {status: 200} + ]}); + } + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + // dest quota GET + src quota GET (different node) + the real POST + assert.equal(ajaxStub.callCount, 3); + assert.equal(tb.uploadStates.length, 1); + assert.equal(tb.uploadStates[0].success, true); + }); + + it('shows a quota usage alert on success once the destination crosses the warning threshold', () => { + var tb = makeTb(); + var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1); + var from = makeItem('file', {name: 'a.txt', size: 150}, 3, 1); + stubAjax(function(opts) { + if (opts.method === 'GET') { + // used(800) + totalSize(150) = 950 <= max(1000) -> not rejected, + // but 950 > max * threshold(0.9) = 900 -> alert should fire. + return {responseJSON: {used: 800, max: 1000, user_guid: 'u1', storage_type: 'NII_STORAGE'}}; + } + if (opts.type === 'POST') { + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 'osfstorage', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'}, + 'success', + {status: 200} + ]}); + } + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined); + assert.equal(growlCallWithType('warning'), true); + }); + + it('creates a new item via tb.createItem for a COPY and posts a copy action', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + var postedData; + stubAjax(function(opts) { + postedData = JSON.parse(opts.data); + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined); + assert.equal(tb.createItem.called, true); + assert.equal(postedData.action, 'copy'); + }); + + it('posts a rename action for a RENAME operation', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'old.txt'}, 3, 9); + var postedData; + stubAjax(function(opts) { + postedData = JSON.parse(opts.data); + return fakeDeferred({status: 'done', args: [ + {name: 'new.txt', provider: 's3compat', path: '/new.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.RENAME, to, from, 'new.txt', undefined); + assert.equal(postedData.action, 'rename'); + assert.equal(postedData.rename, 'new.txt'); + }); + + it('shows a pending growl and stops when the API responds 202', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'done', args: [{}, 'success', {status: 202}]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(growlCallWithType('info'), true); + assert.equal(tb.uploadStates.length, 0); + }); + + it('shows an oversized-file growl and records a failure when the API rejects with 413', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [ + {status: 413, responseJSON: {oversized_files: [{name: 'big.txt', size: 5000000}], max_size: 1000000}}, + 'error' + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(growlStub.called, true); + assert.equal(tb.uploadStates.length, 1); + assert.equal(tb.uploadStates[0].success, false); + }); + + it('shows the mapped MESSAGE_MAP text when the API rejects with 406 and a known message_key', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [ + {status: 406, responseJSON: {message: 'raw message', message_key: 'quota_exceeded'}}, + 'error' + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(growlStub.called, true); + assert.equal(tb.uploadStates[0].success, false); + }); + + it('reports to Raven and shows a fallback message when the API fails with a generic 500', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [{status: 500}, 'error']}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(ravenStub.called, true); + assert.equal(growlStub.called, true); + }); + + // The cases below close the gaps found in the coverage report + // (RDM-osf.io/coverage/html/js/fangorn.js.html) for _findOversizedFiles + // and doItemOp. See investigation/2026-09-03-fangorn-coverage-findoversizedfiles-doitemop.md. + + it('rejects a folder move and reports oversized files found inside it', () => { + var tb = makeTb(); + // Different providers: shouldSkipSizeCheck only shortcuts a same-provider move. + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1); + var from = makeItem('folder', {name: 'myFolder'}, 3, 1); + var smallChild = makeItem('file', {name: 'small.txt', size: 100}, 10, 3); + var bigChild = makeItem('file', {name: 'big.txt', size: 5000000}, 11, 3); + from.children = [smallChild, bigChild]; + ajaxStub = sinon.stub($, 'ajax'); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(ajaxStub.called, false); + assert.equal(growlStub.called, true); + assert.equal(from.inProgress, false); + }); + + it('sums child file sizes when moving a folder and rejects if quota would be exceeded', () => { + var tb = makeTb(); + var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/'}, 5, 1); + var from = makeItem('folder', {name: 'myFolder'}, 3, 1); + var child = makeItem('file', {name: 'a.txt', size: 200}, 10, 3); + from.children = [child]; + stubAjax(function(opts) { + if (opts.method === 'GET') { + return {responseJSON: {used: 900, max: 1000, user_guid: 'u1', storage_type: 'NII_STORAGE'}}; + } + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined); + assert.equal(ajaxStub.callCount, 1); + assert.equal(from.notify.update.called, true); + assert.equal(from.inProgress, false); + }); + + // _rejectConflictItem isn't exported, so these two exercise it through its only + // real entry points (the oversized-file and quota pre-flight checks in doItemOp) + // to cover the object-identity pendingReadyFiles filter and its doSyncMove advance. + it('keeps other pending files queued when a pre-flight reject leaves the sync queue non-empty', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1); + var from = makeItem('file', {name: 'big.txt', size: 5000000}, 3, 1); + var other = makeItem('file', {name: 'other.txt'}, 77, 1); + tb.pendingReadyFiles = [from, other]; + ajaxStub = sinon.stub($, 'ajax'); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, 'replace'); + // Filtered by object identity, not id: "other" survives untouched. + assert.deepEqual(tb.pendingReadyFiles, [other]); + assert.equal(ajaxStub.called, false); + // Queue still has "other" pending, so doSyncMove must not have run yet. + assert.equal(tb.modal.dismiss.callCount, 1); + assert.equal(tb.modal.update.called, false); + }); + + it('advances the sync move queue via doSyncMove once a pre-flight reject empties pendingReadyFiles', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1); + var from = makeItem('file', {name: 'big.txt', size: 5000000}, 3, 1); + tb.pendingReadyFiles = [from]; + ajaxStub = sinon.stub($, 'ajax'); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, 'replace'); + assert.equal(tb.pendingReadyFiles.length, 0); + // Emptying the queue must trigger doSyncMove -> displayMoveStats, which shows + // the move-status modal because addFileStatus already recorded the rejection + // (displayMoveStats resets tb.moveStates back to [] once it builds the modal). + assert.equal(tb.modal.update.called, true); + }); + + it('shows a "conflicts left to resolve" modal when the sync queue has pending conflicts', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + tb.syncFileMoveCache.s3compat = {conflicts: [{item: from, folder: to}], ready: []}; + stubAjax(function() { + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(tb.modal.update.called, true); + }); + + it('preserves the branch and checks the sync-ready queue when moving a github item', () => { + var tb = makeTb(); + // github isn't in the default syncFileMoveCache fixture; add it so the + // notRenameOp conflicts-check doesn't throw on an undefined entry. + tb.syncFileMoveCache.github = {conflicts: [], ready: []}; + var to = makeItem('folder', {provider: 'github', path: '/'}, 5, 1); + var from = makeItem('file', {provider: 'github', name: 'a.txt', branch: 'main'}, 3, 1); + var postedData; + stubAjax(function(opts) { + postedData = JSON.parse(opts.data); + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 'github', path: '/a.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(postedData.branch, 'main'); + }); + + it('removes the existing item at destination when the server responds 200 with a name conflict', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1); + var existingChild = makeItem('file', {name: 'a.txt', provider: 's3compat'}, 99, 5); + to.children = [existingChild]; + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(existingChild.removeSelf.called, true); + }); + + it('rebuilds the child subtree when the moved item is a folder with children in the response', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1); + var from = makeItem('folder', {provider: 's3compat', name: 'myFolder'}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'done', args: [ + { + name: 'myFolder', provider: 's3compat', path: '/dest/myFolder/', kind: 'folder', + nodeUrl: 'https://osf.io/to/', + children: [{name: 'child.txt', kind: 'file', provider: 's3compat'}] + }, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.equal(tb.buildTree.called, true); + assert.equal(from.add.called, true); + assert.equal(from.open, true); + assert.equal(from.load, true); + }); + + it('removes the newly created item when a COPY operation fails', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [{status: 500}, 'error']}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined); + assert.equal(tb.createItem.called, true); + var createdItem = tb.createItem.returnValues[0]; + assert.equal(createdItem.removeSelf.called, true); + }); + + it('shows the responseJSON message for a non-413/406/500/503 failure', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [ + {status: 400, responseJSON: {message: 'custom failure message'}}, + 'error' + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + var calledWithMessage = growlStub.getCalls().some(function(call) { return call.args[1] === 'custom failure message'; }); + assert.equal(calledWithMessage, true); + }); + + it('uses textStatus as the error message on a 503 response', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + stubAjax(function() { + return fakeDeferred({status: 'fail', args: [{status: 503}, 'Service Unavailable']}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + var calledWithMessage = growlStub.getCalls().some(function(call) { return call.args[1] === 'Service Unavailable'; }); + assert.equal(calledWithMessage, true); + }); + + it('removes the settled item from pendingReadyFiles once the operation completes', () => { + var tb = makeTb(); + var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1); + var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1); + var other = makeItem('file', {name: 'other.txt'}, 77, 1); + tb.pendingReadyFiles = [from, other]; + stubAjax(function() { + return fakeDeferred({status: 'done', args: [ + {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'}, + 'success', + {status: 200} + ]}); + }); + Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined); + assert.deepEqual(tb.pendingReadyFiles, [other]); + }); }); describe('FGToolbar IME Keydown Handling', function() { diff --git a/website/templates/emails/file_operation_failed.html.mako b/website/templates/emails/file_operation_failed.html.mako index 2c72fed9b16..e7ab9105d15 100644 --- a/website/templates/emails/file_operation_failed.html.mako +++ b/website/templates/emails/file_operation_failed.html.mako @@ -59,8 +59,30 @@ diff --git a/website/translations/en/LC_MESSAGES/js_messages.po b/website/translations/en/LC_MESSAGES/js_messages.po index 7b54523cf44..32f8b61b92a 100644 --- a/website/translations/en/LC_MESSAGES/js_messages.po +++ b/website/translations/en/LC_MESSAGES/js_messages.po @@ -9475,3 +9475,15 @@ msgstr "" "GakuNin RDM is not responsible for the service or for your use " "thereof.\n" "\n" + +msgid "File「%1$s」is too large (%2$s). Max file size is %3$s." +msgstr "" + +msgid "You do not have enough available quota." +msgstr "" + +msgid "Not enough quota to move/copy the file." +msgstr "" + +msgid "Not enough quota to move/copy. The total size of the folder %1$s." +msgstr "" diff --git a/website/translations/ja/LC_MESSAGES/js_messages.po b/website/translations/ja/LC_MESSAGES/js_messages.po index 8b8775c7c4f..18651932072 100644 --- a/website/translations/ja/LC_MESSAGES/js_messages.po +++ b/website/translations/ja/LC_MESSAGES/js_messages.po @@ -11021,3 +11021,15 @@ msgstr "" "RDMプロジェクトは外部サービスに接続されます。このサービスを利用することで、それら外部サービスの利用規約に拘束されます。GakuNin " "RDMは、それらサービスまたはユーザーによるその利用に対して責任を負いません。\n" "\n" + +msgid "File「%1$s」is too large (%2$s). Max file size is %3$s." +msgstr "ファイル「%1$s」は大きすぎます (%2$s)。最大ファイルサイズは %3$s です。" + +msgid "You do not have enough available quota." +msgstr "利用可能なクォータが不足しています。" + +msgid "Not enough quota to move/copy the file." +msgstr "ファイルを移動/コピーするのに十分なクォータがありません。" + +msgid "Not enough quota to move/copy. The total size of the folder %1$s." +msgstr "移動/コピーするための空き容量が足りません。フォルダのサイズは%1$sです。" diff --git a/website/translations/js_messages.pot b/website/translations/js_messages.pot index f9bba444c1c..bd6c80d03c1 100644 --- a/website/translations/js_messages.pot +++ b/website/translations/js_messages.pot @@ -9314,3 +9314,15 @@ msgid "" "${user} made bibliographic group ${mapcore_groups} a " "non-bibliographic group on ${node}" msgstr "" + +msgid "File「%1$s」is too large (%2$s). Max file size is %3$s." +msgstr "" + +msgid "You do not have enough available quota." +msgstr "" + +msgid "Not enough quota to move/copy the file." +msgstr "" + +msgid "Not enough quota to move/copy. The total size of the folder %1$s." +msgstr "" diff --git a/website/util/quota.py b/website/util/quota.py index 1990073a444..d38d3664e4b 100644 --- a/website/util/quota.py +++ b/website/util/quota.py @@ -142,6 +142,17 @@ def get_project_storage_type(node): @file_signals.file_updated.connect def update_used_quota(self, target, user, event_type, payload): + # FILE_MOVED and FILE_COPIED carry 'source'/'destination', not 'metadata'. + # Handle them first before the metadata_provider guard below. + if event_type in (FileLog.FILE_MOVED, FileLog.FILE_COPIED): + dest_provider = payload.get('destination', {}).get('provider') + if dest_provider == 'osfstorage': + return # Hooks already handled (create_child), SKIP double counting + src_provider = payload.get('source', {}).get('provider') + if src_provider == 'osfstorage' or src_provider in PROVIDERS: + _handle_move_copy(event_type, target, user, payload) + return + data = dict(payload.get('metadata')) if payload.get('metadata') else None metadata_provider = data.get('provider') if payload.get('metadata') else None if metadata_provider == 'osfstorage' or metadata_provider in PROVIDERS: @@ -180,6 +191,9 @@ def update_used_quota(self, target, user, event_type, payload): storage_type = get_project_storage_type(target) if event_type == FileLog.FILE_ADDED: + metadata_provider = (payload.get('metadata') or {}).get('provider') + if metadata_provider == 'osfstorage': + return file_added(target, payload, file_node, storage_type) elif event_type == FileLog.FILE_REMOVED: if metadata_provider in PROVIDERS: @@ -223,76 +237,31 @@ def file_added(target, payload, file_node, storage_type): file_size = int(payload['metadata']['size']) if file_size < 0: return - try: - if check_select_for_update(): - user_quota = UserQuota.objects.filter( - user=target.creator, - storage_type=storage_type - ).select_for_update().get() - else: - user_quota = UserQuota.objects.get( - user=target.creator, - storage_type=storage_type - ) - user_quota.used += file_size - user_quota.save() - except UserQuota.DoesNotExist: + update_quota(target, file_size, storage_type, add=True) + FileInfo.objects.create(file=file_node, file_size=file_size) + +def node_removed(target, user, payload, file_node, storage_type): + if 'osf.trashed' not in file_node.type: + logging.error('FileNode is not trashed, cannot update used quota!') + return + + total_removed = 0 + for removed_file in get_node_file_list(file_node): try: - with transaction.atomic(): - UserQuota.objects.create( - user=target.creator, - storage_type=storage_type, - max_quota=api_settings.DEFAULT_MAX_QUOTA, - used=file_size - ) - except IntegrityError: if check_select_for_update(): - user_quota = UserQuota.objects.filter( - user=target.creator, - storage_type=storage_type - ).select_for_update().get() + file_info = FileInfo.objects.filter(file=removed_file).select_for_update().get() else: - user_quota = UserQuota.objects.get( - user=target.creator, - storage_type=storage_type - ) - user_quota.used += file_size - user_quota.save() - - FileInfo.objects.create(file=file_node, file_size=file_size) + file_info = FileInfo.objects.get(file=removed_file) + except FileInfo.DoesNotExist: + logging.error('FileInfo not found, cannot update used quota!') + continue -def node_removed(target, user, payload, file_node, storage_type): - if check_select_for_update(): - user_quota = UserQuota.objects.filter( - user=target.creator, - storage_type=storage_type - ).select_for_update().first() - else: - user_quota = UserQuota.objects.filter( - user=target.creator, - storage_type=storage_type - ).first() - if user_quota is not None: - if 'osf.trashed' not in file_node.type: - logging.error('FileNode is not trashed, cannot update used quota!') - return + total_removed += file_info.file_size + file_info.file_size = 0 + file_info.save() - for removed_file in get_node_file_list(file_node): - try: - if check_select_for_update(): - file_info = FileInfo.objects.filter(file=removed_file).select_for_update().get() - else: - file_info = FileInfo.objects.get(file=removed_file) - except FileInfo.DoesNotExist: - logging.error('FileInfo not found, cannot update used quota!') - continue - - user_quota.used -= file_info.file_size - if user_quota.used < 0: - user_quota.used = 0 - file_info.file_size = 0 - file_info.save() - user_quota.save() + if total_removed > 0: + update_quota(target, total_removed, storage_type, add=False) def file_modified(target, user, payload, file_node, storage_type): file_size = int(payload['metadata']['size']) @@ -390,3 +359,126 @@ def get_node_file_list(file_node): file_list.append(child_file_node) return file_list + +def _handle_move_copy(event_type, target, user, payload): + """Resolve the destination file_node and dispatch to file_moved or file_copied. + + For FILE_MOVED / FILE_COPIED events the relevant file information lives in + ``payload['destination']``, not in ``payload['metadata']``. Only extended-storage + providers (PROVIDERS) destinations are processed here. + + Scenarios handled: + - osfstorage → extended storage (quota decreases for source node) + - same provider moves / copies (if it involves extended storage) + + (Note: Moves TO osfstorage are handled entirely by OSF hooks, bypassing this flow). + + :param str event_type: FileLog.FILE_MOVED or FileLog.FILE_COPIED + :param AbstractNode target: destination project node + :param OSFUser user: acting user + :param dict payload: full webhook payload + """ + + if event_type == FileLog.FILE_MOVED: + file_moved(target, payload) + +def file_moved(target, payload): + """Update per-user-per-storage used quota when moving file + + :param Object target: Id of project + :param dict payload: The payload of the move operation + """ + if isinstance(target, AbstractNode): + file_size = -1 + dest_size = payload['destination'].get('size', None) + children = payload['destination'].get('children', None) + # Move file + if dest_size is not None: + file_size = int(dest_size) + # Move folder + elif children is not None: + file_size = get_file_size(payload['destination']) + if file_size < 0: + return + + source_node_id = payload['source']['nid'] + source_node = AbstractNode.objects.get(guids___id=source_node_id) + # Move from bulk-mount + if payload['source']['provider'] == 'osfstorage' \ + and source_node.type != 'osf.quickfilesnode': + source_storage_type = get_project_storage_type(source_node) + update_quota(source_node, file_size, source_storage_type, add=False) + +def update_quota(target, file_size, storage_type, add=True): + """Add or subtract file_size from UserQuota.used for target.creator. + + This is the single source of truth for UserQuota updates. + Both ``file_added`` and ``node_removed`` delegate to this function. + + :param target: AbstractNode whose creator's quota is updated + :param int file_size: size in bytes to add or subtract + :param int storage_type: UserQuota.NII_STORAGE or CUSTOM_STORAGE + :param bool add: True to add (upload), False to subtract (delete) + """ + try: + if check_select_for_update(): + user_quota = UserQuota.objects.filter( + user=target.creator, + storage_type=storage_type + ).select_for_update().get() + else: + user_quota = UserQuota.objects.get( + user=target.creator, + storage_type=storage_type + ) + if add: + user_quota.used += file_size + else: + user_quota.used -= file_size + if user_quota.used < 0: + user_quota.used = 0 + user_quota.save() + except UserQuota.DoesNotExist: + if not add: + return + try: + with transaction.atomic(): + UserQuota.objects.create( + user=target.creator, + storage_type=storage_type, + max_quota=api_settings.DEFAULT_MAX_QUOTA, + used=file_size + ) + except IntegrityError: + if check_select_for_update(): + user_quota = UserQuota.objects.filter( + user=target.creator, + storage_type=storage_type + ).select_for_update().get() + else: + user_quota = UserQuota.objects.get( + user=target.creator, + storage_type=storage_type + ) + user_quota.used += file_size + user_quota.save() + +def get_file_size(children): + children = children.get('children', []) + size = 0 + for child in children: + if child['kind'] == 'file': + try: + child_size = int(child['size']) + except (TypeError, ValueError): + child_size = None + if child_size is None or child_size < 0: + logger.warning( + 'child size is %r for %r; treating size as 0 for quota calculation', + child_size, child.get('name') + ) + child_size = 0 + size += int(child_size) + else: + size += get_file_size(child) + return size
- An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'}. - Please log in and try this action again. If the problem persists, please email ${osf_support_email}. + <% + ei = context.get('error_info') or {} + err_type = ei.get('type', '') + %> + % if err_type == 'quota_exceeded': + An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'} because you do not have enough available storage quota. + Please free up storage space or contact your administrator to increase your quota, then try again. If the problem persists, please email ${osf_support_email}. + % elif err_type == 'oversized': + <% + oversized_files = ei.get('oversized_files', []) + max_size_bytes = ei.get('max_size') or 0 + max_size_mb = max_size_bytes // (1024 * 1024) if max_size_bytes else 0 + %> + An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'} because the following file(s) exceed the maximum allowed file size${' of {} MB'.format(max_size_mb) if max_size_mb else ''}: +
    + % for f in oversized_files: +
  • ${f.get('name', 'Unknown file')} (${'{:.1f}'.format(f.get('size', 0) / (1024 * 1024))} MB)
    • + % endfor +
+ Please reduce the file size and try again. If the problem persists, please email ${osf_support_email}. + % else: + An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'}. + Please log in and try this action again. If the problem persists, please email ${osf_support_email}. + % endif